00010000:00000010:1.1F:1775764671.807214:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994210a04700. 00010000:00000010:1.1:1775764671.807256:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994201482600. 00010000:00000010:1.1:1775764671.917887:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994210a04700. 00010000:00000010:1.1:1775764671.917904:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994200031c00. 00010000:00000010:1.1:1775764671.917909:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420b14be00. 00000080:00000010:1.1:1775764671.933139:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275fb180. 00000080:00000010:1.1:1775764671.933175:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207573f60. 00010000:00000010:1.1:1775764671.946213:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994210a04700. 00010000:00000010:1.1:1775764671.946251:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994201483e00. 00000080:00000010:1.1:1775764671.971088:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227606b40. 00000080:00000010:1.1:1775764671.971112:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942074f3080. 00010000:00000010:1.1:1775764672.574209:0:12575:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994210a04700. 00010000:00000010:1.1:1775764672.578085:0:12575:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420b14a400. 00000080:00000010:1.1:1775764672.585934:0:12575:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227610000. 00000080:00000010:1.1:1775764672.585968:0:12575:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472840. 00010000:00000010:1.1:1775764672.616108:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994210a04700. 00010000:00000010:1.1:1775764672.616135:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994201482000. 00000080:00000010:1.1:1775764672.840195:0:12575:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227616b40. 00000080:00000010:1.1:1775764672.840210:0:12575:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942075737e0. 00010000:00000010:1.1:1775764672.846536:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994210a04700. 00010000:00000010:1.1:1775764672.846573:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994201482400. 00000080:00000010:1.1:1775764672.863244:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227615280. 00000080:00000010:1.1:1775764672.863262:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942200bfa20. 00000080:00000010:1.1:1775764673.129987:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227614200. 00000080:00000010:1.1:1775764673.130004:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8ea0. 00010000:00000010:1.1:1775764673.130014:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994210a04700. 00010000:00000010:1.1:1775764673.130020:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994201483c00. 00000080:00000010:1.1:1775764673.137079:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227611080. 00000080:00000010:1.1:1775764673.137110:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942200bf9c0. 00010000:00000010:1.1:1775764673.150305:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994210a04700. 00010000:00000010:1.1:1775764673.150318:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994201482600. 00000080:00000010:1.1:1775764673.153705:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942276118c0. 00000080:00000010:1.1:1775764673.158414:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942200bf9e0. 00010000:00000010:1.1:1775764673.158440:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994210a04d00. 00010000:00000010:1.1:1775764673.158448:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994201483c00. 00000080:00000010:1.1:1775764673.163221:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227613180. 00000080:00000010:1.1:1775764673.163242:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994209d6c180. 00010000:00000010:1.1:1775764673.457357:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994210a04700. 00010000:00000010:1.1:1775764673.457388:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420b14b600. 00000080:00000010:1.1:1775764673.865496:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942276298c0. 00000080:00000010:1.1:1775764673.865529:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8380. 00010000:00000010:1.1:1775764673.870953:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994210a04700. 00010000:00000010:1.1:1775764673.870976:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994201482600. 00010000:00000010:1.1:1775764674.002637:0:12577:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99421263ac00. 00010000:00000010:1.1:1775764721.912499:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994218f6fa00. 00000080:00000010:1.1:1775764722.142169:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6ae00. 00010000:00000010:1.1:1775764722.144193:0:14450:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994218f6f000. 00010000:00000010:1.1:1775764722.160960:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994218f6e200. 00010000:00000010:1.1:1775764722.252246:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075db000. 00010000:00000010:1.1:1775764723.667775:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209c33d00. 00010000:00000010:1.1:1775764723.667790:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942013d3c00. 00000080:00000010:1.1:1775764724.706276:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8da0. 00010000:00000010:1.1:1775764724.823210:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99421263a000. 00010000:00000010:1.1:1775764725.005799:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075dae00. 00010000:00000010:1.1:1775764725.042206:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994210a04e00. 00010000:00000010:1.1:1775764725.042217:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994218f6e800. 00000080:00000010:1.1:1775764725.105369:0:20:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994209598500. 00000080:00000010:1.1:1775764725.105388:0:20:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472ea0. 00000080:00000010:1.1:1775764725.105395:0:20:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472da0. 00000080:00000010:1.1:1775764725.105398:0:20:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472280. 00000080:00000010:1.1:1775764725.105401:0:20:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994209598fe0. 00000080:00000010:1.1:1775764725.105408:0:20:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472760. 00000080:00000010:1.1:1775764725.105412:0:20:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472ae0. 00000080:00000010:1.1:1775764725.105415:0:20:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472360. 00000080:00000010:1.1:1775764725.105421:0:20:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8a80. 00000080:00000010:1.1:1775764725.105425:0:20:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8160. 00010000:00000010:1.1:1775764725.108631:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99421263ae00. 00010000:00000010:1.1:1775764725.108646:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075db200. 00000080:00000010:1.1:1775764725.173208:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8320. 00000080:00000010:1.1:1775764725.342443:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a7a0. 00010000:00000010:1.1:1775764725.342461:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99421263ad00. 00010000:00000010:1.1:1775764725.342466:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075dba00. 00010000:00000010:1.1:1775764725.814269:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994220375600. 00010000:00000010:1.1:1775764725.814277:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994200030800. 00010000:00000010:1.1:1775764727.651499:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994209968400. 00010000:00000010:1.1:1775764727.704439:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075cbc00. 00010000:00000010:1.1:1775764728.015629:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994209969c00. 00010000:00000010:1.1:1775764728.202326:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075cac00. 00010000:00000010:1.1:1775764728.506230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994203fb0600. 00010000:00000010:1.1:1775764728.890314:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075cba00. 00010000:00000010:1.1:1775764729.108712:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075cba00. 00010000:00000010:1.1:1775764729.191223:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994203fb0800. 00000080:00000010:1.1:1775764729.191236:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994209598980. 00010000:00000010:1.1:1775764729.200887:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994210a04a00. 00010000:00000010:1.1:1775764729.719352:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207534600. 00000080:00000010:1.1:1775764732.776616:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8180. 00010000:00000010:1.1:1775764732.875404:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994209969000. 00010000:00000010:1.1:1775764733.033316:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994210812600. 00010000:00000010:1.1:1775764733.033337:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942420c0400. 00000080:00000010:1.1:1775764733.033345:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8b80. 00010000:00000010:1.1:1775764733.042713:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994211cc1200. 00010000:00000010:1.1:1775764733.042724:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207939a00. 00000080:00000010:1.1:1775764733.071266:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a6e0. 00010000:00000010:1.1:1775764733.368609:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207938000. 00000080:00000010:1.1:1775764733.368618:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a660. 00010000:00000010:1.1:1775764733.587200:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793bc00. 00000080:00000010:1.1:1775764733.587209:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472660. 00010000:00000010:1.1:1775764733.596219:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994211cc1f00. 00010000:00000010:1.1:1775764734.102565:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793b000. 00000080:00000010:1.1:1775764734.114182:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472960. 00010000:00000010:1.1:1775764735.140603:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207939200. 00000080:00000010:1.1:1775764735.140627:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6af80. 00010000:00000010:1.1:1775764735.152740:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994211cc1700. 00010000:00000010:1.1:1775764735.715769:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca1000. 00010000:00000010:1.1:1775764735.856202:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207535400. 00010000:00000010:1.1:1775764736.139039:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994211cc1700. 00000080:00000010:1.1:1775764736.139054:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a500. 00010000:00000010:1.1:1775764736.139067:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207938800. 00000080:00000010:1.1:1775764736.139072:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a820. 00010000:00000010:1.1:1775764736.139076:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994211cc1d00. 00010000:00000010:1.1:1775764736.165425:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207938200. 00000080:00000010:1.1:1775764736.169610:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a920. 00010000:00000010:1.1:1775764736.169646:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994211cc1d00. 00000080:00000010:1.1:1775764736.554526:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8d80. 00010000:00000010:1.1:1775764736.566237:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994211cc1d00. 00010000:00000010:1.1:1775764736.566249:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207939200. 00000080:00000010:1.1:1775764736.591352:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a840. 00010000:00000010:1.1:1775764736.591371:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994211cc1700. 00010000:00000010:1.1:1775764736.661288:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420694c600. 00000080:00000010:1.1:1775764736.781263:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8dc0. 00010000:00000010:1.1:1775764736.781279:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207939200. 00010000:00000010:1.1:1775764736.974435:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994211cc1d00. 00000080:00000010:1.1:1775764736.974455:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6aa20. 00010000:00000010:1.1:1775764737.039053:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207938600. 00000080:00000010:1.1:1775764737.039072:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6aec0. 00010000:00000010:1.1:1775764737.129875:0:6159:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994201483e00. 00010000:00000010:1.1:1775764737.259230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6000. 00010000:00000010:1.1:1775764737.640828:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994219da2200. 00010000:00000010:1.1:1775764737.640848:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420694d600. 00010000:00000010:1.1:1775764737.818487:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ce1600. 00010000:00000010:1.1:1775764737.941453:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207938800. 00010000:00000010:1.1:1775764737.989730:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ce1a00. 00010000:00000010:1.1:1775764738.145194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6e00. 00010000:00000010:1.1:1775764738.374937:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ce1e00. 00000080:00000010:1.1:1775764738.374950:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6ae80. 00010000:00000010:1.1:1775764738.374962:0:2393:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207eef500. 00010000:00000010:1.1:1775764738.633541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ce1600. 00000080:00000010:1.1:1775764738.638478:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6af60. 00010000:00000010:1.1:1775764738.656245:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207eef500. 00010000:00000010:1.1:1775764739.398230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a6c00. 00010000:00000010:1.1:1775764739.853337:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994209db2c00. 00010000:00000010:1.1:1775764740.665818:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075daa00. 00000080:00000010:1.1:1775764740.965337:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994209598180. 00000080:00000010:1.1:1775764741.880431:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756fce0. 00010000:00000010:1.1:1775764741.927209:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994205348600. 00010000:00000010:1.1:1775764742.311549:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cbca00. 00010000:00000010:1.1:1775764742.373018:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cbd800. 00010000:00000010:1.1:1775764743.860931:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1800. 00010000:00000010:1.1:1775764743.931909:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075cba00. 00010000:00000010:1.1:1775764743.931930:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cbca00. 00000080:00000010:1.1:1775764744.406604:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942095988e0. 00010000:00000010:1.1:1775764744.634261:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8200. 00010000:00000010:1.1:1775764744.653137:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075ca800. 00000080:00000010:1.1:1775764744.817211:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275eb180. 00010000:00000010:1.1:1775764744.926112:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917800. 00010000:00000010:1.1:1775764744.926167:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916a00. 00000080:00000010:1.1:1775764745.298268:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6ac40. 00010000:00000010:1.1:1775764745.302209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0800. 00010000:00000010:1.1:1775764745.302217:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917600. 00000080:00000010:1.1:1775764745.369921:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756fa80. 00010000:00000010:1.1:1775764745.645355:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916c00. 00010000:00000010:1.1:1775764745.754314:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917200. 00000080:00000010:1.1:1775764745.926086:0:13471:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c17460. 00000080:00000010:1.1:1775764746.464594:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994209598580. 00010000:00000010:1.1:1775764746.464622:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0c00. 00000080:00000010:1.1:1775764746.980113:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a820. 00010000:00000010:1.1:1775764746.984260:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1800. 00010000:00000010:1.1:1775764747.063984:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916a00. 00000080:00000010:1.1:1775764747.101832:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c171e0. 00010000:00000010:1.1:1775764747.157513:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8600. 00010000:00000010:1.1:1775764747.157521:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb9e00. 00010000:00000010:1.1:1775764747.209184:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8c00. 00010000:00000010:1.1:1775764747.300382:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916800. 00010000:00000010:1.1:1775764747.300388:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916400. 00000080:00000010:1.1:1775764747.335172:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c17c20. 00010000:00000010:1.1:1775764747.413601:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916000. 00010000:00000010:1.1:1775764747.469823:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8600. 00010000:00000010:1.1:1775764747.516667:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207eeff00. 00010000:00000010:1.1:1775764747.516675:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0c00. 00010000:00000010:1.1:1775764747.535253:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0c00. 00000080:00000010:1.1:1775764747.535260:0:2392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6af40. 00010000:00000010:1.1:1775764747.738564:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0a00. 00010000:00000010:1.1:1775764747.743828:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917200. 00010000:00000010:1.1:1775764747.743834:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916e00. 00010000:00000010:1.1:1775764748.073190:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917800. 00000080:00000010:1.1:1775764748.089378:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a6e0. 00010000:00000010:1.1:1775764748.091440:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917600. 00000080:00000010:1.1:1775764748.143362:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c17aa0. 00000080:00000010:1.1:1775764748.273581:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c17f60. 00010000:00000010:1.1:1775764748.294352:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916e00. 00010000:00000010:1.1:1775764748.294361:0:2393:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420550b200. 00010000:00000010:1.1:1775764748.294365:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cbd800. 00010000:00000010:1.1:1775764748.694394:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0400. 00010000:00000010:1.1:1775764748.694434:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb3e00. 00010000:00000010:1.1:1775764749.205300:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994205348600. 00010000:00000010:1.1:1775764749.205315:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb9c00. 00000080:00000010:1.1:1775764749.376975:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a8a0. 00010000:00000010:1.1:1775764749.405792:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917200. 00010000:00000010:1.1:1775764749.505688:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb3e00. 00000080:00000010:1.1:1775764749.568424:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994209598700. 00010000:00000010:1.1:1775764749.970339:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420550be00. 00010000:00000010:1.1:1775764749.970355:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb9c00. 00000080:00000010:1.1:1775764749.994369:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994209598b00. 00000080:00000010:1.1:1775764750.004297:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff99422762b180. 00010000:00000010:1.1:1775764750.500375:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0e00. 00010000:00000010:1.1:1775764750.505014:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8600. 00000080:00000010:1.1:1775764751.006608:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a3a0. 00000080:00000010:1.1:1775764751.425198:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6ad60. 00010000:00000010:1.1:1775764751.493045:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0e00. 00010000:00000010:1.1:1775764751.493059:0:2392:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943b00. 00010000:00000010:1.1:1775764751.493065:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917a00. 00000080:00000010:1.1:1775764751.493071:0:2392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6aa20. 00000080:00000010:1.1:1775764751.493101:0:2392:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942189ab9c0. 00010000:00000010:1.1:1775764752.086668:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916000. 00010000:00000010:1.1:1775764752.261190:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917600. 00000080:00000010:1.1:1775764752.523375:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c17680. 00000080:00000010:1.1:1775764752.523385:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c17360. 00010000:00000010:1.1:1775764752.688201:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917800. 00010000:00000010:1.1:1775764752.885365:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916600. 00010000:00000010:1.1:1775764753.111720:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917600. 00000080:00000010:1.1:1775764753.168973:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942095988e0. 00010000:00000010:1.1:1775764753.335435:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994203fb1600. 00010000:00000010:1.1:1775764753.648022:0:12856:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0a00. 00010000:00000010:1.1:1775764753.859408:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1600. 00000080:00000010:1.1:1775764753.859419:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a3a0. 00010000:00000010:1.1:1775764753.992596:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994200030800. 00010000:00000010:1.1:1775764754.072736:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0800. 00000080:00000010:1.1:1775764754.367354:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c17e40. 00000080:00000010:1.1:1775764754.420012:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c17240. 00000080:00000010:1.1:1775764754.420041:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275dd280. 00010000:00000010:1.1:1775764754.433590:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0600. 00000080:00000010:1.1:1775764754.433609:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6aae0. 00010000:00000010:1.1:1775764754.464377:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb9e00. 00000080:00000010:1.1:1775764754.732843:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c172e0. 00010000:00000010:1.1:1775764754.732857:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903400. 00000080:00000010:1.1:1775764754.872704:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c17b00. 00010000:00000010:1.1:1775764754.872720:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1000. 00000080:00000010:1.1:1775764754.950462:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c170a0. 00010000:00000010:1.1:1775764755.317692:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207953c00. 00010000:00000010:1.1:1775764755.614898:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207953000. 00010000:00000010:1.1:1775764755.628198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0800. 00010000:00000010:1.1:1775764755.628215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0a00. 00010000:00000010:1.1:1775764755.990859:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb3600. 00010000:00000010:1.1:1775764756.053500:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2a00. 00010000:00000010:1.1:1775764756.078014:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2800. 00010000:00000010:1.1:1775764756.129695:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902400. 00010000:00000010:1.1:1775764756.321236:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902e00. 00010000:00000010:1.1:1775764756.418565:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903400. 00000080:00000010:1.1:1775764756.531581:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756faa0. 00010000:00000010:1.1:1775764756.552455:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0c00. 00000080:00000010:1.1:1775764756.552470:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6afc0. 00000080:00000010:1.1:1775764756.648270:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f200. 00000080:00000010:1.1:1775764756.648293:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a280. 00010000:00000010:1.1:1775764757.376427:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209897a00. 00010000:00000010:1.1:1775764757.376444:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207938800. 00010000:00000010:1.1:1775764757.984226:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420550b800. 00010000:00000010:1.1:1775764757.984236:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0200. 00010000:00000010:1.1:1775764757.997464:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902e00. 00010000:00000010:1.1:1775764758.107325:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2800. 00010000:00000010:1.1:1775764758.112355:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207953c00. 00010000:00000010:1.1:1775764758.112360:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902200. 00000080:00000010:1.1:1775764758.112367:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f300. 00000080:00000010:1.1:1775764758.112372:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756fd00. 00010000:00000010:1.1:1775764758.156573:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207953a00. 00010000:00000010:1.1:1775764758.156578:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1600. 00000080:00000010:1.1:1775764758.276667:0:1358:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c17360. 00010000:00000010:1.1:1775764758.276691:0:1358:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902600. 00010000:00000010:1.1:1775764758.276697:0:1358:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966800. 00010000:00000010:1.1:1775764758.276700:0:1358:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902800. 00010000:00000010:1.1:1775764758.378593:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0c00. 00010000:00000010:1.1:1775764758.735184:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902e00. 00010000:00000010:1.1:1775764758.827534:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da800. 00010000:00000010:1.1:1775764758.942978:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902c00. 00010000:00000010:1.1:1775764759.212306:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970600. 00010000:00000010:1.1:1775764759.212312:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902400. 00000080:00000010:1.1:1775764759.329968:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c177a0. 00000080:00000010:1.1:1775764759.329985:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f1c0. 00010000:00000010:1.1:1775764759.644662:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903600. 00000080:00000010:1.1:1775764759.741336:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a0e0. 00010000:00000010:1.1:1775764759.741348:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903600. 00010000:00000010:1.1:1775764759.838212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dae00. 00010000:00000010:1.1:1775764759.861050:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dbe00. 00000080:00000010:1.1:1775764759.934283:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f380. 00010000:00000010:1.1:1775764759.955207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dbe00. 00010000:00000010:1.1:1775764760.143366:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dae00. 00000080:00000010:1.1:1775764760.412101:0:2392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994209598ec0. 00010000:00000010:1.1:1775764760.412125:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970e00. 00010000:00000010:1.1:1775764760.482209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903a00. 00000080:00000010:1.1:1775764760.496489:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f260. 00010000:00000010:1.1:1775764760.499308:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902400. 00010000:00000010:1.1:1775764760.515313:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895400. 00000080:00000010:1.1:1775764760.555761:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c17500. 00010000:00000010:1.1:1775764760.605202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902600. 00010000:00000010:1.1:1775764760.770202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902200. 00010000:00000010:1.1:1775764760.813277:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db000. 00010000:00000010:1.1:1775764760.845170:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894600. 00010000:00000010:1.1:1775764760.870878:0:2401:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902400. 00010000:00000010:1.1:1775764760.938321:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895400. 00010000:00000010:1.1:1775764760.980555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079daa00. 00010000:00000010:1.1:1775764761.055540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ce1200. 00000080:00000010:1.1:1775764761.055548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c17ca0. 00010000:00000010:1.1:1775764761.079211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902400. 00010000:00000010:1.1:1775764761.313204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789c600. 00010000:00000010:1.1:1775764761.374931:0:12856:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970a00. 00010000:00000010:1.1:1775764761.878230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970e00. 00000080:00000010:1.1:1775764761.886346:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c17f60. 00000080:00000010:1.1:1775764762.048497:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c17300. 00010000:00000010:1.1:1775764762.053422:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971000. 00010000:00000010:1.1:1775764762.164112:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971a00. 00000080:00000010:1.1:1775764762.164126:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a3a0. 00000080:00000010:1.1:1775764762.164135:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c17b60. 00000080:00000010:1.1:1775764762.173122:0:2392:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994218995ac0. 00000080:00000010:1.1:1775764762.215088:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c17fc0. 00010000:00000010:1.1:1775764762.231649:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789c800. 00010000:00000010:1.1:1775764762.251885:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789da00. 00000080:00000010:1.1:1775764762.359919:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c17860. 00010000:00000010:1.1:1775764762.359927:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789d400. 00010000:00000010:1.1:1775764762.359931:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ad200. 00010000:00000010:1.1:1775764762.359935:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789c000. 00010000:00000010:1.1:1775764762.359939:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971600. 00000080:00000010:1.1:1775764762.359942:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6afc0. 00000080:00000010:1.1:1775764762.359953:0:14454:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994218994a40. 00010000:00000010:1.1:1775764762.371320:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970600. 00010000:00000010:1.1:1775764762.371339:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994205348600. 00010000:00000010:1.1:1775764762.371344:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da800. 00010000:00000010:1.1:1775764762.392352:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894e00. 00000080:00000010:1.1:1775764762.415981:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f7a0. 00000080:00000010:1.1:1775764762.416005:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff99422762dac0. 00010000:00000010:1.1:1775764762.416012:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dbc00. 00010000:00000010:1.1:1775764762.445085:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895600. 00010000:00000010:1.1:1775764762.473926:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895a00. 00010000:00000010:1.1:1775764762.773189:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420550b200. 00010000:00000010:1.1:1775764762.773209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903200. 00010000:00000010:1.1:1775764762.864274:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da200. 00010000:00000010:1.1:1775764763.216265:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895200. 00000080:00000010:1.1:1775764763.216273:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994209598000. 00000080:00000010:1.1:1775764763.441584:0:6159:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c17940. 00010000:00000010:1.1:1775764763.441608:0:6159:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970c00. 00000080:00000010:1.1:1775764763.521259:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942095983e0. 00010000:00000010:1.1:1775764763.760756:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da000. 00010000:00000010:1.1:1775764764.042662:0:2400:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db400. 00010000:00000010:1.1:1775764764.061109:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894600. 00010000:00000010:1.1:1775764764.156120:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789dc00. 00000080:00000010:1.1:1775764764.338264:0:9316:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994209598480. 00010000:00000010:1.1:1775764764.338308:0:9316:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789d000. 00010000:00000010:1.1:1775764764.548473:0:2401:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789d600. 00000080:00000010:1.1:1775764764.725678:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756fc40. 00010000:00000010:1.1:1775764764.781975:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895400. 00010000:00000010:1.1:1775764764.790889:0:2400:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994206989500. 00010000:00000010:1.1:1775764764.797665:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7000. 00010000:00000010:1.1:1775764765.161649:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971000. 00010000:00000010:1.1:1775764765.201076:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970000. 00010000:00000010:1.1:1775764765.257919:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789d600. 00000080:00000010:1.1:1775764765.278596:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942095983a0. 00000080:00000010:1.1:1775764765.278606:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a580. 00010000:00000010:1.1:1775764765.278613:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789c800. 00010000:00000010:1.1:1775764765.476248:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789d000. 00010000:00000010:1.1:1775764765.532317:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789d000. 00010000:00000010:1.1:1775764765.598870:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789c800. 00010000:00000010:1.1:1775764765.648360:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970800. 00010000:00000010:1.1:1775764765.843392:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789dc00. 00010000:00000010:1.1:1775764765.916733:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0600. 00010000:00000010:1.1:1775764765.991610:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789d000. 00010000:00000010:1.1:1775764766.187038:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894400. 00010000:00000010:1.1:1775764766.635203:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078adc00. 00010000:00000010:1.1:1775764766.635235:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789c000. 00010000:00000010:1.1:1775764766.635240:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943800. 00010000:00000010:1.1:1775764766.635244:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789d800. 00010000:00000010:1.1:1775764766.689226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789dc00. 00010000:00000010:1.1:1775764766.713308:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0400. 00000080:00000010:1.1:1775764766.796415:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a840. 00010000:00000010:1.1:1775764767.442577:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789d000. 00000080:00000010:1.1:1775764767.673205:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c17d20. 00000080:00000010:1.1:1775764767.783194:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f9c0. 00010000:00000010:1.1:1775764767.915575:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0400. 00010000:00000010:1.1:1775764767.950535:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1000. 00010000:00000010:1.1:1775764768.147924:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da400. 00010000:00000010:1.1:1775764768.634664:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da600. 00010000:00000010:1.1:1775764768.768316:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970200. 00000080:00000010:1.1:1775764768.865512:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a5c0. 00010000:00000010:1.1:1775764768.865535:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895400. 00010000:00000010:1.1:1775764769.192512:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1000. 00010000:00000010:1.1:1775764769.192527:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971800. 00010000:00000010:1.1:1775764769.212525:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970e00. 00010000:00000010:1.1:1775764769.225214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dbe00. 00010000:00000010:1.1:1775764769.726199:0:2400:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db600. 00010000:00000010:1.1:1775764769.932265:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903000. 00010000:00000010:1.1:1775764770.098796:0:2400:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902400. 00010000:00000010:1.1:1775764770.199052:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902600. 00010000:00000010:1.1:1775764770.523937:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dbe00. 00000080:00000010:1.1:1775764770.810424:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c174a0. 00000080:00000010:1.1:1775764770.810452:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994209598180. 00000080:00000010:1.1:1775764771.188204:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275dc200. 00000080:00000010:1.1:1775764771.711827:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f340. 00010000:00000010:1.1:1775764771.711860:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dbe00. 00010000:00000010:1.1:1775764771.711868:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078adc00. 00010000:00000010:1.1:1775764771.711873:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1a00. 00010000:00000010:1.1:1775764771.711876:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209897f00. 00010000:00000010:1.1:1775764771.711904:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789c200. 00010000:00000010:1.1:1775764771.711908:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943000. 00010000:00000010:1.1:1775764771.711912:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895600. 00010000:00000010:1.1:1775764771.711919:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420788e400. 00010000:00000010:1.1:1775764771.711923:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789d800. 00010000:00000010:1.1:1775764771.923966:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dba00. 00010000:00000010:1.1:1775764772.018211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dbc00. 00010000:00000010:1.1:1775764772.050333:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db800. 00010000:00000010:1.1:1775764772.225837:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079cb200. 00010000:00000010:1.1:1775764772.297240:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971e00. 00010000:00000010:1.1:1775764772.328607:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902a00. 00000080:00000010:1.1:1775764772.450223:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f580. 00010000:00000010:1.1:1775764772.459215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dbc00. 00000080:00000010:1.1:1775764772.527368:0:2392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a620. 00010000:00000010:1.1:1775764772.600419:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dbe00. 00010000:00000010:1.1:1775764772.636512:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079cb000. 00010000:00000010:1.1:1775764772.672407:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971e00. 00010000:00000010:1.1:1775764772.735803:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db600. 00010000:00000010:1.1:1775764773.021580:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079cae00. 00010000:00000010:1.1:1775764773.078192:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903a00. 00010000:00000010:1.1:1775764773.100210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dbc00. 00010000:00000010:1.1:1775764773.367209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903a00. 00010000:00000010:1.1:1775764773.481278:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994208aff500. 00010000:00000010:1.1:1775764773.481305:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb9e00. 00000080:00000010:1.1:1775764773.850515:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f620. 00010000:00000010:1.1:1775764773.871661:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dbc00. 00010000:00000010:1.1:1775764774.028952:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079cac00. 00010000:00000010:1.1:1775764774.088888:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994200030800. 00010000:00000010:1.1:1775764774.150592:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903200. 00000080:00000010:1.1:1775764774.536964:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c179e0. 00010000:00000010:1.1:1775764775.477178:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902a00. 00010000:00000010:1.1:1775764775.890391:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789dc00. 00000080:00000010:1.1:1775764776.006781:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c178a0. 00010000:00000010:1.1:1775764776.038969:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1a00. 00010000:00000010:1.1:1775764776.038983:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420788eb00. 00010000:00000010:1.1:1775764776.038988:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789d600. 00010000:00000010:1.1:1775764776.111204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079cb600. 00010000:00000010:1.1:1775764776.385186:0:36:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902200. 00010000:00000010:1.1:1775764776.447232:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0a00. 00010000:00000010:1.1:1775764776.459543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079ca200. 00010000:00000010:1.1:1775764776.801592:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00010000:00000010:1.1:1775764776.894190:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079daa00. 00010000:00000010:1.1:1775764776.997594:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916600. 00010000:00000010:1.1:1775764777.181215:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207c60600. 00010000:00000010:1.1:1775764777.181231:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca0000. 00010000:00000010:1.1:1775764777.670043:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079cae00. 00010000:00000010:1.1:1775764777.717359:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079cac00. 00010000:00000010:1.1:1775764777.773202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079cae00. 00010000:00000010:1.1:1775764777.846214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971a00. 00010000:00000010:1.1:1775764777.898404:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079caa00. 00010000:00000010:1.1:1775764778.141326:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a7800. 00010000:00000010:1.1:1775764778.317190:0:2401:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079caa00. 00000080:00000010:1.1:1775764778.538391:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942200bf9e0. 00000080:00000010:1.1:1775764778.546753:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994209598040. 00000080:00000010:1.1:1775764778.546762:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f400. 00010000:00000010:1.1:1775764778.546769:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a7000. 00010000:00000010:1.1:1775764778.546775:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994208aff600. 00010000:00000010:1.1:1775764778.546780:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898400. 00010000:00000010:1.1:1775764778.546783:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079cac00. 00000080:00000010:1.1:1775764778.557071:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f780. 00000080:00000010:1.1:1775764778.557109:0:14454:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d9080. 00010000:00000010:1.1:1775764778.567199:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898400. 00000080:00000010:1.1:1775764778.567207:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6afe0. 00010000:00000010:1.1:1775764778.705207:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a7000. 00010000:00000010:1.1:1775764779.374479:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898800. 00010000:00000010:1.1:1775764779.385266:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899e00. 00010000:00000010:1.1:1775764779.385288:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899200. 00000080:00000010:1.1:1775764779.385338:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6ab60. 00000080:00000010:1.1:1775764779.385345:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a5c0. 00000080:00000010:1.1:1775764779.385348:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6ad60. 00010000:00000010:1.1:1775764779.385352:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dac00. 00000080:00000010:1.1:1775764779.412235:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a720. 00000080:00000010:1.1:1775764779.412245:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c177a0. 00000080:00000010:1.1:1775764779.412253:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994209598140. 00010000:00000010:1.1:1775764779.412260:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899a00. 00000080:00000010:1.1:1775764779.629837:0:13:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c17840. 00010000:00000010:1.1:1775764779.629851:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da400. 00000080:00000010:1.1:1775764779.797788:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c17a40. 00000080:00000010:1.1:1775764780.161307:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a340. 00000080:00000010:1.1:1775764780.506369:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756fae0. 00000080:00000010:1.1:1775764780.777287:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a980. 00000080:00000010:1.1:1775764780.777317:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275deb40. 00010000:00000010:1.1:1775764780.782894:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da000. 00010000:00000010:1.1:1775764780.782903:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942069a5a00. 00010000:00000010:1.1:1775764780.782907:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895e00. 00000080:00000010:1.1:1775764780.821578:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c17e00. 00000080:00000010:1.1:1775764780.821608:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275dd280. 00010000:00000010:1.1:1775764781.107498:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895000. 00010000:00000010:1.1:1775764781.221741:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb9c00. 00000080:00000010:1.1:1775764781.221751:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6afc0. 00010000:00000010:1.1:1775764781.361219:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994210812000. 00010000:00000010:1.1:1775764781.361226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994203fb0200. 00010000:00000010:1.1:1775764781.456237:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2a00. 00010000:00000010:1.1:1775764781.681294:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dbe00. 00010000:00000010:1.1:1775764781.681354:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895200. 00000080:00000010:1.1:1775764782.208804:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f700. 00010000:00000010:1.1:1775764782.229079:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2400. 00000080:00000010:1.1:1775764782.229103:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994209598280. 00010000:00000010:1.1:1775764782.375204:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895000. 00000080:00000010:1.1:1775764782.401706:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756fc60. 00010000:00000010:1.1:1775764782.645727:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dbc00. 00010000:00000010:1.1:1775764782.666776:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2400. 00010000:00000010:1.1:1775764782.669188:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895800. 00010000:00000010:1.1:1775764782.834515:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994205348b00. 00010000:00000010:1.1:1775764782.834539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dbc00. 00010000:00000010:1.1:1775764783.317951:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895400. 00010000:00000010:1.1:1775764783.327186:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789d200. 00010000:00000010:1.1:1775764783.327203:0:2393:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942069a5a00. 00010000:00000010:1.1:1775764783.327209:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894400. 00010000:00000010:1.1:1775764783.425726:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db800. 00010000:00000010:1.1:1775764783.561382:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2400. 00010000:00000010:1.1:1775764783.561399:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894200. 00000080:00000010:1.1:1775764783.821808:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c17040. 00000080:00000010:1.1:1775764783.821836:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff99422762eb40. 00010000:00000010:1.1:1775764783.833647:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789d000. 00000080:00000010:1.1:1775764783.833664:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6ad80. 00010000:00000010:1.1:1775764783.908712:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789c200. 00000080:00000010:1.1:1775764784.572219:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c17bc0. 00010000:00000010:1.1:1775764784.917259:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079ca200. 00010000:00000010:1.1:1775764784.964507:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079ca200. 00000080:00000010:1.1:1775764784.964516:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c17080. 00000080:00000010:1.1:1775764785.242495:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756fda0. 00000080:00000010:1.1:1775764785.242529:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994218997380. 00010000:00000010:1.1:1775764785.253221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2e00. 00000080:00000010:1.1:1775764785.253237:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942095985a0. 00010000:00000010:1.1:1775764785.362363:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916600. 00010000:00000010:1.1:1775764785.509493:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a6600. 00000080:00000010:1.1:1775764785.509522:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6ac40. 00010000:00000010:1.1:1775764785.881541:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917e00. 00010000:00000010:1.1:1775764785.933716:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079cbc00. 00010000:00000010:1.1:1775764785.933724:0:2393:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420788ea00. 00010000:00000010:1.1:1775764785.933727:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db800. 00010000:00000010:1.1:1775764786.110745:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2400. 00010000:00000010:1.1:1775764786.258563:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970e00. 00010000:00000010:1.1:1775764786.315022:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079cbe00. 00010000:00000010:1.1:1775764786.315050:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970c00. 00010000:00000010:1.1:1775764786.348287:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994218766200. 00010000:00000010:1.1:1775764786.348370:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075cac00. 00010000:00000010:1.1:1775764786.439482:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971a00. 00010000:00000010:1.1:1775764786.454206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970e00. 00000080:00000010:1.1:1775764786.454251:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a3a0. 00000080:00000010:1.1:1775764786.454260:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c17f00. 00000080:00000010:1.1:1775764786.454266:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c17380. 00000080:00000010:1.1:1775764786.454270:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c17920. 00000080:00000010:1.1:1775764786.454273:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6aec0. 00000080:00000010:1.1:1775764786.454277:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a880. 00000080:00000010:1.1:1775764786.454281:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f0e0. 00010000:00000010:1.1:1775764786.454286:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075da800. 00010000:00000010:1.1:1775764787.008838:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894200. 00010000:00000010:1.1:1775764787.268286:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994218fd6700. 00010000:00000010:1.1:1775764787.268311:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902400. 00000080:00000010:1.1:1775764787.364865:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756fc60. 00010000:00000010:1.1:1775764787.501210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895800. 00010000:00000010:1.1:1775764787.751441:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971e00. 00010000:00000010:1.1:1775764787.761615:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894600. 00010000:00000010:1.1:1775764788.259906:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994201483600. 00000080:00000010:1.1:1775764788.509691:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c17b40. 00000080:00000010:1.1:1775764788.509710:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942189939c0. 00010000:00000010:1.1:1775764788.719180:0:6158:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078b3300. 00010000:00000010:1.1:1775764788.719188:0:6158:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079caa00. 00010000:00000010:1.1:1775764788.729249:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420788e500. 00010000:00000010:1.1:1775764788.729263:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dac00. 00010000:00000010:1.1:1775764788.729268:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207c60200. 00010000:00000010:1.1:1775764788.729272:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789c200. 00010000:00000010:1.1:1775764788.729276:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994208aff900. 00010000:00000010:1.1:1775764788.729279:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079ca200. 00010000:00000010:1.1:1775764788.729304:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994211bdb800. 00010000:00000010:1.1:1775764788.729308:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970e00. 00010000:00000010:1.1:1775764788.736000:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078b3800. 00010000:00000010:1.1:1775764788.736016:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079cac00. 00010000:00000010:1.1:1775764788.738199:0:2400:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078b3e00. 00010000:00000010:1.1:1775764788.738210:0:2400:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079ca800. 00010000:00000010:1.1:1775764788.738217:0:2400:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994218fd6000. 00010000:00000010:1.1:1775764788.738223:0:2400:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0800. 00010000:00000010:1.1:1775764788.738230:0:2400:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ad700. 00010000:00000010:1.1:1775764788.738236:0:2400:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903400. 00010000:00000010:1.1:1775764788.738244:0:2400:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078b3900. 00010000:00000010:1.1:1775764788.738249:0:2400:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789d800. 00000080:00000010:1.1:1775764788.877201:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c172c0. 00000080:00000010:1.1:1775764788.948516:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994209598fa0. 00010000:00000010:1.1:1775764789.032555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dba00. 00000080:00000010:1.1:1775764789.032577:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994209598260. 00010000:00000010:1.1:1775764789.159204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da400. 00010000:00000010:1.1:1775764789.159213:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994208afff00. 00010000:00000010:1.1:1775764789.159219:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971800. 00010000:00000010:1.1:1775764789.253549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da400. 00010000:00000010:1.1:1775764789.253563:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994208aff900. 00010000:00000010:1.1:1775764789.253567:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970e00. 00010000:00000010:1.1:1775764789.311091:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970e00. 00010000:00000010:1.1:1775764789.314794:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894000. 00010000:00000010:1.1:1775764789.421283:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994203fb0000. 00010000:00000010:1.1:1775764789.434233:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994203fb1200. 00010000:00000010:1.1:1775764790.232408:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8000. 00010000:00000010:1.1:1775764790.283543:0:14461:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970600. 00010000:00000010:1.1:1775764790.303935:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966d00. 00010000:00000010:1.1:1775764790.303943:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8800. 00010000:00000010:1.1:1775764790.309540:0:2393:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943000. 00010000:00000010:1.1:1775764790.309556:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994209db2a00. 00010000:00000010:1.1:1775764790.446466:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899c00. 00000080:00000010:1.1:1775764790.545888:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6ab60. 00010000:00000010:1.1:1775764790.570281:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899400. 00010000:00000010:1.1:1775764790.570297:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb9c00. 00010000:00000010:1.1:1775764790.817202:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966d00. 00010000:00000010:1.1:1775764790.817217:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da400. 00010000:00000010:1.1:1775764791.005743:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899000. 00010000:00000010:1.1:1775764791.038399:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8000. 00000080:00000010:1.1:1775764791.050337:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c17240. 00010000:00000010:1.1:1775764791.050352:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899a00. 00010000:00000010:1.1:1775764791.187087:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899600. 00000080:00000010:1.1:1775764791.187098:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c179a0. 00000080:00000010:1.1:1775764791.209225:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c172e0. 00010000:00000010:1.1:1775764791.702320:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970600. 00010000:00000010:1.1:1775764791.714475:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899000. 00000080:00000010:1.1:1775764791.714498:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c172c0. 00000080:00000010:1.1:1775764792.217190:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6abc0. 00000080:00000010:1.1:1775764792.217213:0:14454:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994218996300. 00010000:00000010:1.1:1775764792.254627:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ce0800. 00000080:00000010:1.1:1775764792.254635:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6aac0. 00010000:00000010:1.1:1775764792.373546:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789d800. 00000080:00000010:1.1:1775764792.373644:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994209598920. 00000080:00000010:1.1:1775764793.164966:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f2c0. 00010000:00000010:1.1:1775764793.198054:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a7000. 00000080:00000010:1.1:1775764793.198080:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a760. 00010000:00000010:1.1:1775764793.413460:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894600. 00010000:00000010:1.1:1775764793.562288:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899400. 00010000:00000010:1.1:1775764793.651220:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207534800. 00000080:00000010:1.1:1775764794.837571:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994209598040. 00010000:00000010:1.1:1775764794.878849:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903a00. 00010000:00000010:1.1:1775764794.899261:0:1358:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902600. 00010000:00000010:1.1:1775764795.021197:0:2393:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994218fd6700. 00010000:00000010:1.1:1775764795.021208:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903400. 00000080:00000010:1.1:1775764795.031587:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c17780. 00010000:00000010:1.1:1775764795.138240:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dba00. 00010000:00000010:1.1:1775764795.232111:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902400. 00010000:00000010:1.1:1775764795.251868:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da400. 00010000:00000010:1.1:1775764795.380054:0:2401:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ef800. 00010000:00000010:1.1:1775764795.380064:0:2401:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207535c00. 00000080:00000010:1.1:1775764795.392336:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f840. 00000080:00000010:1.1:1775764795.392372:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d6b40. 00010000:00000010:1.1:1775764795.404648:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902200. 00010000:00000010:1.1:1775764795.459224:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789d800. 00010000:00000010:1.1:1775764795.481766:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903200. 00010000:00000010:1.1:1775764795.497203:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917e00. 00010000:00000010:1.1:1775764795.544639:0:2392:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078efa00. 00010000:00000010:1.1:1775764795.544653:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902400. 00000080:00000010:1.1:1775764795.554754:0:2392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a860. 00000080:00000010:1.1:1775764795.554788:0:2392:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d6300. 00010000:00000010:1.1:1775764795.649355:0:6159:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894600. 00000080:00000010:1.1:1775764795.698887:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d0000. 00010000:00000010:1.1:1775764795.775049:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902200. 00010000:00000010:1.1:1775764795.775104:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb9c00. 00000080:00000010:1.1:1775764795.909621:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f220. 00010000:00000010:1.1:1775764795.990325:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ce1600. 00010000:00000010:1.1:1775764795.990335:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420794c000. 00010000:00000010:1.1:1775764795.990339:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a6200. 00000080:00000010:1.1:1775764796.019058:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a440. 00000080:00000010:1.1:1775764796.019090:0:14454:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994218992940. 00010000:00000010:1.1:1775764796.340451:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2a00. 00010000:00000010:1.1:1775764796.597210:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902200. 00000080:00000010:1.1:1775764796.597220:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a6a0. 00000080:00000010:1.1:1775764796.800363:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994209598780. 00000080:00000010:1.1:1775764796.800390:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994218994200. 00010000:00000010:1.1:1775764797.085472:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c400. 00000080:00000010:1.1:1775764797.097265:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a3a0. 00000080:00000010:1.1:1775764797.296443:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c17f40. 00000080:00000010:1.1:1775764797.296492:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994218991080. 00010000:00000010:1.1:1775764797.553382:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81a00. 00010000:00000010:1.1:1775764797.570387:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81600. 00010000:00000010:1.1:1775764797.660820:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798dc00. 00010000:00000010:1.1:1775764798.054392:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ce1600. 00010000:00000010:1.1:1775764798.084400:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81200. 00000080:00000010:1.1:1775764798.084408:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c17580. 00010000:00000010:1.1:1775764798.186130:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078a1000. 00010000:00000010:1.1:1775764798.186152:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80400. 00000080:00000010:1.1:1775764798.232243:0:2400:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c17380. 00010000:00000010:1.1:1775764798.256182:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798ce00. 00010000:00000010:1.1:1775764798.256359:0:2393:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078a1500. 00010000:00000010:1.1:1775764798.256364:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81200. 00010000:00000010:1.1:1775764798.256368:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c600. 00010000:00000010:1.1:1775764798.339829:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d88e00. 00010000:00000010:1.1:1775764798.339880:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793c600. 00010000:00000010:1.1:1775764798.456335:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d89c00. 00010000:00000010:1.1:1775764798.577847:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d88600. 00010000:00000010:1.1:1775764798.583247:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ce1600. 00010000:00000010:1.1:1775764799.013349:0:2401:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d000. 00010000:00000010:1.1:1775764799.013404:0:2401:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81a00. 00010000:00000010:1.1:1775764799.046852:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966600. 00010000:00000010:1.1:1775764799.046863:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798cc00. 00000080:00000010:1.1:1775764799.069116:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8000. 00010000:00000010:1.1:1775764799.081672:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207c60600. 00010000:00000010:1.1:1775764799.081686:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075dbe00. 00010000:00000010:1.1:1775764799.307912:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994218f6f000. 00010000:00000010:1.1:1775764799.421390:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798ce00. 00000080:00000010:1.1:1775764799.421400:0:2392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a9a0. 00010000:00000010:1.1:1775764799.754474:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d400. 00010000:00000010:1.1:1775764799.775763:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c400. 00010000:00000010:1.1:1775764799.775831:0:20:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942069a5a00. 00010000:00000010:1.1:1775764799.775837:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d89800. 00010000:00000010:1.1:1775764799.775861:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dba00. 00000080:00000010:1.1:1775764799.778239:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6abc0. 00000080:00000010:1.1:1775764799.778265:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994209598260. 00010000:00000010:1.1:1775764799.788435:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81a00. 00010000:00000010:1.1:1775764799.817346:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dba00. 00000080:00000010:1.1:1775764799.817368:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c17760. 00000080:00000010:1.1:1775764799.963451:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d6b40. 00010000:00000010:1.1:1775764800.001560:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793a200. 00010000:00000010:1.1:1775764800.115250:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793a200. 00010000:00000010:1.1:1775764800.212323:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d89800. 00010000:00000010:1.1:1775764800.691308:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cbcc00. 00010000:00000010:1.1:1775764800.708210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798cc00. 00000080:00000010:1.1:1775764800.708276:0:20:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c179a0. 00010000:00000010:1.1:1775764800.708286:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81a00. 00010000:00000010:1.1:1775764801.215368:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d88e00. 00010000:00000010:1.1:1775764801.266207:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209897a00. 00010000:00000010:1.1:1775764801.266218:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d400. 00000080:00000010:1.1:1775764801.277038:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6afc0. 00000080:00000010:1.1:1775764801.277065:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227601080. 00010000:00000010:1.1:1775764801.301321:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d400. 00000080:00000010:1.1:1775764801.301331:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a780. 00010000:00000010:1.1:1775764801.376700:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798ca00. 00000080:00000010:1.1:1775764801.376728:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c173e0. 00010000:00000010:1.1:1775764801.376735:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420b14a400. 00000080:00000010:1.1:1775764801.414752:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994218991080. 00010000:00000010:1.1:1775764801.592522:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d89800. 00000080:00000010:1.1:1775764801.614184:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994209598fe0. 00010000:00000010:1.1:1775764801.623371:0:2392:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207c70100. 00010000:00000010:1.1:1775764801.623389:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075dbe00. 00010000:00000010:1.1:1775764801.708179:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc800. 00010000:00000010:1.1:1775764801.898367:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6c00. 00010000:00000010:1.1:1775764801.909944:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7000. 00000080:00000010:1.1:1775764801.976622:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994209598fe0. 00000080:00000010:1.1:1775764801.976661:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994209598260. 00010000:00000010:1.1:1775764801.976667:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994218f6f000. 00000080:00000010:1.1:1775764802.001677:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d4200. 00000080:00000010:1.1:1775764802.219767:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c17080. 00000080:00000010:1.1:1775764802.219795:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994218993180. 00000080:00000010:1.1:1775764802.477711:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f3a0. 00000080:00000010:1.1:1775764802.482648:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994218995ac0. 00000080:00000010:1.1:1775764802.637203:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f3a0. 00010000:00000010:1.1:1775764802.697093:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7c00. 00010000:00000010:1.1:1775764802.925214:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d20500. 00010000:00000010:1.1:1775764802.925222:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc600. 00010000:00000010:1.1:1775764802.950304:0:1358:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d20600. 00010000:00000010:1.1:1775764802.950315:0:1358:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903a00. 00010000:00000010:1.1:1775764803.120942:0:2401:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994206989e00. 00010000:00000010:1.1:1775764803.120951:0:2401:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6e00. 00010000:00000010:1.1:1775764803.120955:0:2401:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420b307e00. 00010000:00000010:1.1:1775764803.120958:0:2401:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a6200. 00010000:00000010:1.1:1775764803.137293:0:14461:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420794c100. 00010000:00000010:1.1:1775764803.137302:0:14461:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895800. 00010000:00000010:1.1:1775764803.137307:0:14461:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc000. 00010000:00000010:1.1:1775764803.149818:0:2393:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078b3500. 00010000:00000010:1.1:1775764803.149830:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d88000. 00010000:00000010:1.1:1775764803.296100:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7200. 00010000:00000010:1.1:1775764803.343514:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7200. 00010000:00000010:1.1:1775764803.415192:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942079a4f00. 00010000:00000010:1.1:1775764803.415213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca1200. 00010000:00000010:1.1:1775764803.438351:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca1200. 00000080:00000010:1.1:1775764803.438363:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756ff20. 00010000:00000010:1.1:1775764803.488620:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d20100. 00010000:00000010:1.1:1775764803.488637:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903a00. 00010000:00000010:1.1:1775764803.677202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd400. 00010000:00000010:1.1:1775764803.860473:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca0e00. 00010000:00000010:1.1:1775764803.947806:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209897200. 00010000:00000010:1.1:1775764803.947857:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798dc00. 00000080:00000010:1.1:1775764803.954248:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c17780. 00000080:00000010:1.1:1775764803.962280:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227601080. 00010000:00000010:1.1:1775764803.974365:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d000. 00000080:00000010:1.1:1775764803.974384:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a660. 00010000:00000010:1.1:1775764804.070204:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d22600. 00010000:00000010:1.1:1775764804.070215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc400. 00000080:00000010:1.1:1775764804.075244:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f400. 00000080:00000010:1.1:1775764804.209790:0:20:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f220. 00010000:00000010:1.1:1775764804.209806:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6400. 00000080:00000010:1.1:1775764804.395955:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472be0. 00010000:00000010:1.1:1775764804.395966:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca0c00. 00000080:00000010:1.1:1775764804.823406:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d4200. 00010000:00000010:1.1:1775764805.070495:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942069a5a00. 00010000:00000010:1.1:1775764805.070512:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc000. 00000080:00000010:1.1:1775764805.086395:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f1a0. 00000080:00000010:1.1:1775764805.086413:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227604200. 00010000:00000010:1.1:1775764805.097435:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6600. 00000080:00000010:1.1:1775764805.097455:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a980. 00000080:00000010:1.1:1775764805.235847:0:2393:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994218991080. 00010000:00000010:1.1:1775764805.370914:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207ce3700. 00010000:00000010:1.1:1775764805.370924:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81800. 00010000:00000010:1.1:1775764805.404571:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6600. 00000080:00000010:1.1:1775764805.404581:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a800. 00000080:00000010:1.1:1775764805.548415:0:14453:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d4200. 00010000:00000010:1.1:1775764805.565810:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6800. 00010000:00000010:1.1:1775764805.667435:0:2392:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078efe00. 00010000:00000010:1.1:1775764805.667443:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970600. 00000080:00000010:1.1:1775764805.863001:0:2392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942074723a0. 00000080:00000010:1.1:1775764805.868313:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d18c0. 00010000:00000010:1.1:1775764805.893480:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895800. 00010000:00000010:1.1:1775764805.932206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7e00. 00000080:00000010:1.1:1775764806.054181:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942074720a0. 00010000:00000010:1.1:1775764806.361494:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cbce00. 00000080:00000010:1.1:1775764806.370857:0:20:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c173e0. 00000080:00000010:1.1:1775764806.370867:0:20:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c17020. 00010000:00000010:1.1:1775764806.370872:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a6800. 00010000:00000010:1.1:1775764806.527216:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943300. 00010000:00000010:1.1:1775764806.527251:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798dc00. 00000080:00000010:1.1:1775764806.568259:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472de0. 00000080:00000010:1.1:1775764806.568285:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff99422762b9c0. 00010000:00000010:1.1:1775764806.603648:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6000. 00000080:00000010:1.1:1775764806.603661:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6aa40. 00010000:00000010:1.1:1775764806.658218:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6600. 00010000:00000010:1.1:1775764807.048477:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207534800. 00010000:00000010:1.1:1775764807.067492:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207976900. 00010000:00000010:1.1:1775764807.067508:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994201482600. 00010000:00000010:1.1:1775764807.152345:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079cbe00. 00000080:00000010:1.1:1775764807.263182:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c172e0. 00000080:00000010:1.1:1775764807.403688:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472660. 00000080:00000010:1.1:1775764807.647951:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a9e0. 00010000:00000010:1.1:1775764808.092597:0:2392:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d20d00. 00010000:00000010:1.1:1775764808.092607:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca0a00. 00010000:00000010:1.1:1775764808.191122:0:2393:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207976500. 00010000:00000010:1.1:1775764808.191130:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a6600. 00010000:00000010:1.1:1775764808.191134:0:2393:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207976200. 00010000:00000010:1.1:1775764808.191137:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079ca800. 00010000:00000010:1.1:1775764808.191140:0:2393:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ef100. 00010000:00000010:1.1:1775764808.191143:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798da00. 00010000:00000010:1.1:1775764808.230770:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078efe00. 00010000:00000010:1.1:1775764808.230777:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81800. 00000080:00000010:1.1:1775764808.493128:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff99422762a940. 00010000:00000010:1.1:1775764808.493144:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994208affa00. 00010000:00000010:1.1:1775764808.493150:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2800. 00000080:00000010:1.1:1775764808.656181:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c17540. 00000080:00000010:1.1:1775764808.656219:0:14453:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d9080. 00000080:00000010:1.1:1775764808.763826:0:20:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942074727a0. 00010000:00000010:1.1:1775764808.763836:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1800. 00010000:00000010:1.1:1775764808.763840:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994209db2a00. 00010000:00000010:1.1:1775764808.838362:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994203fb0600. 00010000:00000010:1.1:1775764808.838710:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207eefe00. 00010000:00000010:1.1:1775764808.838716:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916c00. 00000080:00000010:1.1:1775764808.886849:0:2392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472100. 00000080:00000010:1.1:1775764808.886891:0:2392:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942189af380. 00010000:00000010:0.1:1775764808.891291:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078b3f00. 00010000:00000010:0.1:1775764808.891320:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903600. 00000080:00000010:1.1:1775764808.937450:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756ffa0. 00010000:00000010:0.1:1775764809.001999:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420788e500. 00010000:00000010:0.1:1775764809.002012:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075cbc00. 00000080:00000010:1.1:1775764809.014320:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c17c40. 00000080:00000010:0.1:1775764809.018264:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942276039c0. 00000080:00000010:0.1:1775764809.080776:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472ac0. 00000080:00000010:0.1:1775764809.080834:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994218994200. 00010000:00000010:0.1:1775764809.117270:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942079b7000. 00010000:00000010:0.1:1775764809.117276:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0400. 00010000:00000010:0.1:1775764809.145208:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916000. 00010000:00000010:0.1:1775764809.145220:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420550ba00. 00010000:00000010:0.1:1775764809.145225:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb3e00. 00000080:00000010:1.1:1775764809.166814:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f7a0. 00000080:00000010:1.1:1775764809.184575:0:2392:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942189ae300. 00000080:00000010:1.1:1775764809.187458:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2520. 00000080:00000010:1.1:1775764809.187468:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994209598920. 00010000:00000010:1.1:1775764809.187475:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994218fd6300. 00010000:00000010:1.1:1775764809.187480:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970e00. 00010000:00000010:1.1:1775764809.187484:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075cac00. 00010000:00000010:0.1:1775764809.207993:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca0a00. 00000080:00000010:0.1:1775764809.208005:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a200. 00000080:00000010:0.1:1775764809.278760:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff99422762e300. 00010000:00000010:0.1:1775764809.289223:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966700. 00010000:00000010:0.1:1775764809.289239:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca0000. 00000080:00000010:1.1:1775764809.317896:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f800. 00000080:00000010:1.1:1775764809.322288:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d1080. 00010000:00000010:0.1:1775764809.330968:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966e00. 00010000:00000010:0.1:1775764809.330975:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8600. 00000080:00000010:0.1:1775764809.360644:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c170c0. 00000080:00000010:0.1:1775764809.389258:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d3180. 00010000:00000010:1.1:1775764809.446244:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917800. 00000080:00000010:1.1:1775764809.450236:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db22c0. 00000080:00000010:1.1:1775764809.450251:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f440. 00010000:00000010:1.1:1775764809.450256:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207c67200. 00010000:00000010:1.1:1775764809.450263:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894400. 00010000:00000010:1.1:1775764809.450265:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994209db3000. 00000080:00000010:1.1:1775764809.537205:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942095987a0. 00000080:00000010:1.1:1775764809.537245:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d2940. 00010000:00000010:1.1:1775764809.537259:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca0c00. 00000080:00000010:1.1:1775764809.537303:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a960. 00000080:00000010:1.1:1775764809.537311:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994206167000. 00000080:00000010:1.1:1775764809.537319:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942087aaee0. 00000080:00000010:1.1:1775764809.537323:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942087aa0c0. 00000080:00000010:1.1:1775764809.537328:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994210b16fe0. 00000080:00000010:1.1:1775764809.537333:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472920. 00000080:00000010:1.1:1775764809.537378:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756fc20. 00000080:00000010:1.1:1775764809.537387:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c17760. 00000080:00000010:1.1:1775764809.537392:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c172e0. 00000080:00000010:1.1:1775764809.537396:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a9a0. 00000080:00000010:1.1:1775764809.537400:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f120. 00000080:00000010:1.1:1775764809.537405:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f260. 00000080:00000010:1.1:1775764809.537409:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f500. 00000080:00000010:1.1:1775764809.537413:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c17f40. 00000080:00000010:1.1:1775764809.537418:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c173e0. 00000080:00000010:1.1:1775764809.537424:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a780. 00000080:00000010:1.1:1775764809.537428:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c17120. 00010000:00000010:1.1:1775764809.537436:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d22200. 00010000:00000010:1.1:1775764809.537442:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798cc00. 00010000:00000010:1.1:1775764809.537445:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420b14a400. 00000080:00000010:1.1:1775764809.615204:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d6b40. 00000080:00000010:0.1:1775764809.646577:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942189939c0. 00000080:00000010:0.1:1775764809.654182:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f040. 00000080:00000010:1.1:1775764809.661807:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227618000. 00010000:00000010:0.1:1775764809.678213:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420950b600. 00010000:00000010:0.1:1775764809.678220:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971e00. 00000080:00000010:0.1:1775764809.687869:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2660. 00010000:00000010:0.1:1775764809.687877:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793d800. 00010000:00000010:0.1:1775764809.687882:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943900. 00010000:00000010:0.1:1775764809.687886:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894c00. 00010000:00000010:0.1:1775764809.687889:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca0c00. 00010000:00000010:0.1:1775764809.695899:0:2393:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942079a4800. 00010000:00000010:0.1:1775764809.695913:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899000. 00000080:00000010:1.1:1775764809.718876:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f6a0. 00000080:00000010:1.1:1775764809.718907:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275db180. 00010000:00000010:0.1:1775764809.734687:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca0000. 00000080:00000010:0.1:1775764809.757516:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2760. 00000080:00000010:0.1:1775764809.757528:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218144620. 00000080:00000010:0.1:1775764809.757534:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218144700. 00000080:00000010:0.1:1775764809.757538:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420001bf60. 00000080:00000010:0.1:1775764809.757544:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994202dfe9e0. 00000080:00000010:0.1:1775764809.757549:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994202dfe960. 00000080:00000010:0.1:1775764809.757554:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994202dfe200. 00000080:00000010:0.1:1775764809.757557:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994202dfe980. 00000080:00000010:0.1:1775764809.757560:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994202dfe5c0. 00000080:00000010:0.1:1775764809.757566:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c17b40. 00000080:00000010:0.1:1775764809.757572:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c17e40. 00000080:00000010:0.1:1775764809.757581:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f880. 00000080:00000010:0.1:1775764809.757587:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f2c0. 00000080:00000010:0.1:1775764809.757591:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f780. 00000080:00000010:0.1:1775764809.757595:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f320. 00000080:00000010:0.1:1775764809.757600:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756fb00. 00000080:00000010:0.1:1775764809.757604:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c172c0. 00010000:00000010:0.1:1775764809.757610:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d22700. 00010000:00000010:0.1:1775764809.757615:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd600. 00010000:00000010:0.1:1775764809.757620:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd800. 00010000:00000010:0.1:1775764809.767763:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079cb400. 00010000:00000010:0.1:1775764809.767776:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420788ec00. 00010000:00000010:0.1:1775764809.767781:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970000. 00000080:00000010:1.1:1775764809.794737:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d5ac0. 00010000:00000010:1.1:1775764809.798229:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca0c00. 00000080:00000010:0.1:1775764809.862598:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c171c0. 00000080:00000010:0.1:1775764809.862642:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d5280. 00010000:00000010:1.1:1775764809.874468:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca0e00. 00010000:00000010:0.1:1775764809.878445:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ef300. 00010000:00000010:0.1:1775764809.878456:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8000. 00010000:00000010:1.1:1775764809.883568:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dae00. 00010000:00000010:1.1:1775764809.883595:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ad100. 00010000:00000010:1.1:1775764809.883599:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789d400. 00000080:00000010:0.1:1775764809.883776:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f920. 00000080:00000010:0.1:1775764809.883802:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d2100. 00010000:00000010:1.1:1775764809.895427:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ef400. 00010000:00000010:1.1:1775764809.895448:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079caa00. 00000080:00000010:0.1:1775764809.899591:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c17ec0. 00000080:00000010:0.1:1775764809.910232:0:2393:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d98c0. 00010000:00000010:1.1:1775764809.922297:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420788e200. 00010000:00000010:1.1:1775764809.922305:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db400. 00000080:00000010:0.1:1775764809.939243:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d8840. 00000080:00000010:1.1:1775764809.939487:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472000. 00010000:00000010:0.1:1775764809.947778:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a7800. 00010000:00000010:0.1:1775764809.947804:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942069a5400. 00010000:00000010:0.1:1775764809.947828:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971000. 00000080:00000010:1.1:1775764810.044271:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227628840. 00010000:00000010:1.1:1775764810.048665:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078efc00. 00010000:00000010:1.1:1775764810.048687:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917600. 00000080:00000010:0.1:1775764810.052231:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994218994a40. 00010000:00000010:0.1:1775764810.052250:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994218fd6700. 00010000:00000010:0.1:1775764810.052256:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902400. 00000080:00000010:0.1:1775764810.098247:0:2392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a340. 00000080:00000010:0.1:1775764810.098278:0:2392:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227602100. 00010000:00000010:0.1:1775764810.123867:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917c00. 00010000:00000010:1.1:1775764810.269256:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6400. 00010000:00000010:1.1:1775764810.364783:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916a00. 00010000:00000010:1.1:1775764810.524308:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca0e00. 00010000:00000010:0.1:1775764810.546260:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917a00. 00010000:00000010:1.1:1775764810.658437:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916a00. 00010000:00000010:0.1:1775764810.699344:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916c00. 00000080:00000010:1.1:1775764810.737197:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2b20. 00010000:00000010:0.1:1775764810.842002:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1e00. 00010000:00000010:1.1:1775764810.859459:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7000. 00010000:00000010:1.1:1775764810.955986:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942079b7000. 00010000:00000010:1.1:1775764810.955996:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902800. 00000080:00000010:1.1:1775764810.960374:0:6159:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472400. 00000080:00000010:1.1:1775764810.960397:0:6159:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff99422762e300. 00010000:00000010:1.1:1775764810.971499:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902600. 00000080:00000010:1.1:1775764810.971518:0:20:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a5a0. 00010000:00000010:1.1:1775764811.023558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902600. 00010000:00000010:1.1:1775764811.023610:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902800. 00010000:00000010:0.1:1775764811.076529:0:14461:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943500. 00010000:00000010:0.1:1775764811.084757:0:14461:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916800. 00010000:00000010:1.1:1775764811.088969:0:2392:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943600. 00010000:00000010:1.1:1775764811.088978:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb3200. 00010000:00000010:0.1:1775764811.355418:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902000. 00010000:00000010:0.1:1775764811.562829:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6400. 00000080:00000010:0.1:1775764811.659598:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f400. 00010000:00000010:0.1:1775764811.675588:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0a00. 00000080:00000010:0.1:1775764811.876589:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2060. 00010000:00000010:0.1:1775764811.944695:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916800. 00000080:00000010:0.1:1775764812.109236:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2dc0. 00010000:00000010:0.1:1775764812.109262:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916a00. 00000080:00000010:1.1:1775764812.171552:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db21e0. 00010000:00000010:1.1:1775764812.171577:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916000. 00010000:00000010:0.1:1775764812.185571:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916800. 00010000:00000010:0.1:1775764812.233824:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6200. 00010000:00000010:1.1:1775764812.291338:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ad400. 00010000:00000010:1.1:1775764812.291347:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0600. 00000080:00000010:0.1:1775764812.291683:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a5e0. 00000080:00000010:1.1:1775764812.304215:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472000. 00000080:00000010:1.1:1775764812.304250:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff99422762a100. 00010000:00000010:1.1:1775764812.343630:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903400. 00000080:00000010:1.1:1775764812.402665:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a0e0. 00010000:00000010:1.1:1775764812.428765:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903400. 00010000:00000010:0.1:1775764812.553205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916a00. 00010000:00000010:1.1:1775764812.557243:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1a00. 00010000:00000010:1.1:1775764812.829223:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1600. 00000080:00000010:1.1:1775764812.933369:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f620. 00000080:00000010:0.1:1775764813.263089:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2b80. 00010000:00000010:0.1:1775764813.263115:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917800. 00010000:00000010:1.1:1775764813.263816:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207938200. 00000080:00000010:0.1:1775764813.396721:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d7380. 00000080:00000010:1.1:1775764813.460655:0:2401:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2e00. 00000080:00000010:1.1:1775764813.485259:0:6159:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472bc0. 00010000:00000010:1.1:1775764813.485280:0:6159:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209d57a00. 00010000:00000010:1.1:1775764813.485285:0:6159:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6a00. 00010000:00000010:1.1:1775764813.529442:0:2401:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1000. 00010000:00000010:0.1:1775764813.529542:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078a1a00. 00010000:00000010:0.1:1775764813.529550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d88200. 00010000:00000010:0.1:1775764813.529554:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994208affc00. 00010000:00000010:0.1:1775764813.529558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994203fb1200. 00010000:00000010:0.1:1775764813.529561:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207938600. 00010000:00000010:0.1:1775764813.537783:0:2392:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078a1500. 00010000:00000010:0.1:1775764813.537806:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d88800. 00010000:00000010:0.1:1775764813.537810:0:2392:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078e9800. 00010000:00000010:0.1:1775764813.537816:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d800. 00010000:00000010:0.1:1775764813.537820:0:2392:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207ce3c00. 00010000:00000010:0.1:1775764813.537824:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994200030800. 00010000:00000010:0.1:1775764813.537829:0:2392:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943d00. 00010000:00000010:0.1:1775764813.537833:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb9e00. 00010000:00000010:0.1:1775764813.537837:0:2392:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966b00. 00010000:00000010:0.1:1775764813.537841:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971800. 00010000:00000010:0.1:1775764813.548180:0:2393:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d22d00. 00010000:00000010:0.1:1775764813.548193:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc600. 00010000:00000010:0.1:1775764813.558195:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d20900. 00010000:00000010:0.1:1775764813.558204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903200. 00010000:00000010:0.1:1775764813.558212:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ef700. 00010000:00000010:0.1:1775764813.558215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079cbc00. 00010000:00000010:0.1:1775764813.558218:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078eff00. 00010000:00000010:0.1:1775764813.558221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80600. 00010000:00000010:0.1:1775764813.558224:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994205348300. 00010000:00000010:0.1:1775764813.558227:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079ca200. 00010000:00000010:0.1:1775764813.613968:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a6600. 00000080:00000010:0.1:1775764813.661527:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db29e0. 00010000:00000010:0.1:1775764813.748207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a6200. 00000080:00000010:0.1:1775764813.759410:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2f60. 00010000:00000010:0.1:1775764813.759430:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a6800. 00010000:00000010:0.1:1775764813.759434:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a6600. 00010000:00000010:0.1:1775764813.794287:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903600. 00010000:00000010:1.1:1775764813.845225:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a7800. 00000080:00000010:1.1:1775764813.947195:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2700. 00010000:00000010:0.1:1775764814.022219:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6e00. 00010000:00000010:0.1:1775764814.025731:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903e00. 00010000:00000010:0.1:1775764814.025748:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7600. 00010000:00000010:1.1:1775764814.049220:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a6800. 00010000:00000010:0.1:1775764814.151380:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6e00. 00010000:00000010:1.1:1775764814.336830:0:2393:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078e9600. 00010000:00000010:1.1:1775764814.336839:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7200. 00000080:00000010:1.1:1775764814.566868:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2b60. 00010000:00000010:1.1:1775764814.566894:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902e00. 00000080:00000010:0.1:1775764814.702269:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2500. 00000080:00000010:1.1:1775764814.771126:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f540. 00000080:00000010:0.1:1775764814.802404:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db24e0. 00010000:00000010:0.1:1775764814.802431:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0c00. 00000080:00000010:0.1:1775764814.844371:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6ada0. 00010000:00000010:0.1:1775764814.844390:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789d400. 00000080:00000010:1.1:1775764814.870278:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6af60. 00010000:00000010:0.1:1775764814.961208:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789ca00. 00010000:00000010:1.1:1775764815.007215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6800. 00010000:00000010:1.1:1775764815.143186:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789d800. 00010000:00000010:1.1:1775764815.147251:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0400. 00010000:00000010:1.1:1775764815.221801:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb3a00. 00010000:00000010:0.1:1775764815.224123:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6c00. 00010000:00000010:1.1:1775764815.240414:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789ce00. 00010000:00000010:1.1:1775764815.267521:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0000. 00000080:00000010:1.1:1775764815.385354:0:2392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756fa00. 00010000:00000010:0.1:1775764815.396909:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903a00. 00010000:00000010:1.1:1775764815.413575:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0e00. 00010000:00000010:1.1:1775764815.413588:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789c200. 00010000:00000010:0.1:1775764815.467637:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6400. 00010000:00000010:0.1:1775764815.512317:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7c00. 00010000:00000010:1.1:1775764815.543319:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7000. 00010000:00000010:1.1:1775764815.579746:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789da00. 00010000:00000010:0.1:1775764815.601383:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789ca00. 00000080:00000010:1.1:1775764815.615674:0:2392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756fcc0. 00010000:00000010:0.1:1775764815.832215:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078e9600. 00010000:00000010:0.1:1775764815.832228:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789cc00. 00000080:00000010:0.1:1775764815.856253:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756fd00. 00000080:00000010:0.1:1775764815.856275:0:14454:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994218993180. 00010000:00000010:0.1:1775764815.878622:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1800. 00000080:00000010:0.1:1775764815.878635:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472e80. 00010000:00000010:0.1:1775764815.984739:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789da00. 00010000:00000010:0.1:1775764816.132384:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943d00. 00010000:00000010:0.1:1775764816.132393:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902800. 00010000:00000010:0.1:1775764816.137211:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420788e300. 00010000:00000010:0.1:1775764816.137244:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789c200. 00010000:00000010:1.1:1775764816.177284:0:2400:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789d400. 00010000:00000010:0.1:1775764816.231840:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7c00. 00000080:00000010:0.1:1775764816.238997:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f340. 00010000:00000010:0.1:1775764816.239017:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0a00. 00000080:00000010:0.1:1775764816.269088:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2da0. 00000080:00000010:1.1:1775764816.358490:0:2392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472100. 00010000:00000010:1.1:1775764816.365415:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903a00. 00010000:00000010:1.1:1775764816.470772:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789cc00. 00000080:00000010:0.1:1775764816.612900:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db20e0. 00010000:00000010:0.1:1775764816.807337:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903800. 00000080:00000010:0.1:1775764816.817526:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a6a0. 00010000:00000010:1.1:1775764816.865977:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1c00. 00000080:00000010:1.1:1775764816.865993:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472da0. 00010000:00000010:0.1:1775764816.902273:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903200. 00010000:00000010:1.1:1775764816.940047:0:2393:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078e9800. 00010000:00000010:1.1:1775764816.940073:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789cc00. 00000080:00000010:0.1:1775764816.946605:0:14453:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994218993180. 00000080:00000010:1.1:1775764816.955696:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2680. 00000080:00000010:1.1:1775764817.044921:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942074729c0. 00000080:00000010:0.1:1775764817.051238:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2f20. 00000080:00000010:0.1:1775764817.067649:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a340. 00010000:00000010:1.1:1775764817.079815:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1800. 00000080:00000010:1.1:1775764817.079828:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472920. 00000080:00000010:0.1:1775764817.153710:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756fc80. 00010000:00000010:1.1:1775764817.176312:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903200. 00010000:00000010:1.1:1775764817.176396:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789d800. 00010000:00000010:0.1:1775764817.413216:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ad300. 00010000:00000010:0.1:1775764817.413233:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1600. 00010000:00000010:0.1:1775764817.483200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903a00. 00000080:00000010:1.1:1775764817.611494:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2ec0. 00010000:00000010:0.1:1775764817.701671:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902c00. 00000080:00000010:1.1:1775764817.703704:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a840. 00010000:00000010:1.1:1775764817.703716:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903e00. 00010000:00000010:0.1:1775764817.771533:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903400. 00010000:00000010:1.1:1775764817.880416:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789cc00. 00010000:00000010:1.1:1775764817.880439:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943c00. 00010000:00000010:1.1:1775764817.880444:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0800. 00010000:00000010:0.1:1775764817.881932:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902c00. 00010000:00000010:0.1:1775764817.920775:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6800. 00010000:00000010:0.1:1775764817.920786:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420788e100. 00010000:00000010:0.1:1775764817.920790:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7000. 00000080:00000010:1.1:1775764817.977706:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756fb00. 00000080:00000010:1.1:1775764817.977749:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994218991080. 00010000:00000010:0.1:1775764818.024636:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7600. 00010000:00000010:0.1:1775764818.051181:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6e00. 00010000:00000010:0.1:1775764818.116485:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6e00. 00000080:00000010:0.1:1775764818.227486:0:6159:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2620. 00010000:00000010:0.1:1775764818.240210:0:1026:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0e00. 00010000:00000010:1.1:1775764818.240318:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903600. 00010000:00000010:0.1:1775764818.322204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80600. 00010000:00000010:1.1:1775764818.332528:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789ce00. 00010000:00000010:0.1:1775764818.388950:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81400. 00010000:00000010:0.1:1775764818.418236:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7000. 00010000:00000010:0.1:1775764818.427625:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81000. 00000080:00000010:0.1:1775764818.462378:0:2392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2bc0. 00000080:00000010:1.1:1775764818.543095:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472920. 00010000:00000010:1.1:1775764818.543115:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1600. 00000080:00000010:0.1:1775764818.747804:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2000. 00010000:00000010:0.1:1775764818.845716:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81a00. 00000080:00000010:0.1:1775764818.845733:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2d80. 00000080:00000010:1.1:1775764818.855312:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2180. 00010000:00000010:1.1:1775764818.878514:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6c00. 00010000:00000010:1.1:1775764818.887430:0:19:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0000. 00010000:00000010:1.1:1775764818.927872:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80000. 00010000:00000010:1.1:1775764818.935491:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6000. 00010000:00000010:1.1:1775764818.935497:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1000. 00010000:00000010:1.1:1775764818.965703:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81a00. 00000080:00000010:1.1:1775764818.965714:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db20a0. 00010000:00000010:1.1:1775764818.986234:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903c00. 00010000:00000010:1.1:1775764819.125120:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078e9300. 00010000:00000010:1.1:1775764819.125138:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0600. 00010000:00000010:1.1:1775764819.125142:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078e9200. 00010000:00000010:1.1:1775764819.125145:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6400. 00010000:00000010:0.1:1775764819.130683:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420788e200. 00010000:00000010:0.1:1775764819.130690:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7400. 00010000:00000010:0.1:1775764819.162447:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80a00. 00000080:00000010:1.1:1775764819.286389:0:2392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6ad60. 00010000:00000010:1.1:1775764819.286407:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7c00. 00000080:00000010:0.1:1775764819.354372:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a280. 00000080:00000010:1.1:1775764819.482330:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a6a0. 00010000:00000010:1.1:1775764819.482348:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903800. 00010000:00000010:0.1:1775764819.607523:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81a00. 00010000:00000010:0.1:1775764820.117397:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81200. 00010000:00000010:0.1:1775764820.121209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80c00. 00010000:00000010:1.1:1775764820.311739:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903c00. 00010000:00000010:1.1:1775764820.374714:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80c00. 00010000:00000010:1.1:1775764820.387206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902400. 00010000:00000010:0.1:1775764820.429531:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0800. 00000080:00000010:0.1:1775764820.429552:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942074729c0. 00010000:00000010:1.1:1775764820.438563:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6c00. 00010000:00000010:0.1:1775764820.479147:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0800. 00000080:00000010:0.1:1775764820.572947:0:2392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2100. 00000080:00000010:0.1:1775764820.711200:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472760. 00000080:00000010:0.1:1775764820.960412:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472d80. 00010000:00000010:0.1:1775764820.967502:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6600. 00000080:00000010:0.1:1775764821.035385:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a760. 00010000:00000010:1.1:1775764821.047629:0:2401:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902a00. 00010000:00000010:0.1:1775764821.048181:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903800. 00000080:00000010:0.1:1775764821.081367:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db27e0. 00010000:00000010:1.1:1775764821.171303:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6a00. 00000080:00000010:1.1:1775764821.176569:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2b00. 00010000:00000010:1.1:1775764821.176583:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942079b7000. 00010000:00000010:1.1:1775764821.176587:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6600. 00010000:00000010:1.1:1775764821.191280:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7e00. 00000080:00000010:0.1:1775764821.303351:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2a80. 00010000:00000010:1.1:1775764821.380352:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1000. 00010000:00000010:0.1:1775764821.428016:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6e00. 00010000:00000010:1.1:1775764821.442463:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0400. 00010000:00000010:1.1:1775764821.442477:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7600. 00010000:00000010:0.1:1775764821.521123:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1800. 00010000:00000010:1.1:1775764821.533342:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6a00. 00010000:00000010:0.1:1775764821.575773:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6600. 00010000:00000010:1.1:1775764821.618297:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903200. 00000080:00000010:1.1:1775764821.622469:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f7a0. 00000080:00000010:1.1:1775764821.622486:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472760. 00010000:00000010:1.1:1775764821.622493:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7a00. 00010000:00000010:1.1:1775764821.725412:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903800. 00010000:00000010:0.1:1775764821.748093:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ada00. 00010000:00000010:0.1:1775764821.748114:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902200. 00010000:00000010:0.1:1775764821.780440:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6a00. 00010000:00000010:3.1:1775764821.791675:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7e00. 00010000:00000010:0.1:1775764821.889336:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6a00. 00010000:00000010:3.1:1775764821.904034:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207c67500. 00010000:00000010:3.1:1775764821.904057:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6e00. 00000080:00000010:0.1:1775764821.912640:0:2392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2540. 00010000:00000010:0.1:1775764821.912668:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7e00. 00010000:00000010:3.1:1775764821.933228:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7e00. 00010000:00000010:0.1:1775764821.959554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7e00. 00010000:00000010:0.1:1775764822.013278:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6a00. 00000080:00000010:0.1:1775764822.025728:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a0e0. 00010000:00000010:0.1:1775764822.025749:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0a00. 00010000:00000010:3.1:1775764822.025932:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903800. 00010000:00000010:3.1:1775764822.045017:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6400. 00010000:00000010:3.1:1775764822.052413:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0a00. 00000080:00000010:3.1:1775764822.052422:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472400. 00000080:00000010:3.1:1775764822.090408:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2de0. 00000080:00000010:3.1:1775764822.111729:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2e80. 00010000:00000010:0.1:1775764822.123332:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6400. 00010000:00000010:0.1:1775764822.139670:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7e00. 00010000:00000010:3.1:1775764822.193320:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6800. 00010000:00000010:3.1:1775764822.233364:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994201483600. 00010000:00000010:1.1:1775764822.245930:0:2401:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0400. 00010000:00000010:3.1:1775764822.252215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994201483e00. 00000080:00000010:3.1:1775764822.304314:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2a80. 00010000:00000010:1.1:1775764822.309114:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994201483600. 00010000:00000010:1.1:1775764822.309132:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994201483e00. 00010000:00000010:0.1:1775764822.311112:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994201482600. 00010000:00000010:3.1:1775764822.319302:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c200. 00010000:00000010:3.1:1775764822.324549:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902400. 00010000:00000010:3.1:1775764822.324557:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7c00. 00000080:00000010:3.1:1775764822.334205:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2d60. 00000080:00000010:3.1:1775764822.334220:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2540. 00000080:00000010:0.1:1775764822.337377:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a420. 00010000:00000010:0.1:1775764822.460761:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6800. 00000080:00000010:0.1:1775764822.465116:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f040. 00000080:00000010:3.1:1775764822.465207:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472340. 00010000:00000010:1.1:1775764822.465671:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7e00. 00010000:00000010:0.1:1775764822.471598:0:2400:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d600. 00010000:00000010:3.1:1775764822.485275:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6a00. 00000080:00000010:3.1:1775764822.485287:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f2c0. 00010000:00000010:3.1:1775764822.535437:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6a00. 00010000:00000010:3.1:1775764822.535488:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d800. 00010000:00000010:3.1:1775764822.535493:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7400. 00010000:00000010:3.1:1775764822.535497:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798cc00. 00000080:00000010:3.1:1775764822.535503:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f2c0. 00000080:00000010:3.1:1775764822.535521:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756ffa0. 00010000:00000010:3.1:1775764822.566749:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0800. 00010000:00000010:3.1:1775764822.566763:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798de00. 00000080:00000010:0.1:1775764822.596718:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2580. 00010000:00000010:1.1:1775764822.611203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0600. 00010000:00000010:0.1:1775764822.615182:0:1026:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0e00. 00000080:00000010:0.1:1775764822.615190:0:1026:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472bc0. 00010000:00000010:3.1:1775764822.615393:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798da00. 00010000:00000010:1.1:1775764822.695371:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903800. 00010000:00000010:1.1:1775764822.735540:0:14464:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903200. 00010000:00000010:1.1:1775764822.735548:0:14464:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0000. 00000080:00000010:1.1:1775764822.735554:0:14464:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6abc0. 00000080:00000010:1.1:1775764822.735565:0:14464:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942074728a0. 00010000:00000010:0.1:1775764822.741346:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903800. 00010000:00000010:1.1:1775764822.747547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798ca00. 00010000:00000010:3.1:1775764822.794283:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903200. 00010000:00000010:3.1:1775764822.855315:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798ce00. 00000080:00000010:3.1:1775764822.855326:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2640. 00010000:00000010:3.1:1775764822.899082:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7000. 00010000:00000010:3.1:1775764822.899257:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d000. 00010000:00000010:1.1:1775764822.923479:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798ce00. 00010000:00000010:3.1:1775764822.940969:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7000. 00010000:00000010:1.1:1775764822.954963:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207c67500. 00010000:00000010:1.1:1775764822.954972:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6c00. 00000080:00000010:1.1:1775764822.975546:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f2c0. 00000080:00000010:1.1:1775764822.975583:0:2393:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942189918c0. 00010000:00000010:1.1:1775764822.982215:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902a00. 00000080:00000010:1.1:1775764822.982258:0:2392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6ae40. 00010000:00000010:3.1:1775764823.042902:0:1026:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7000. 00010000:00000010:1.1:1775764823.049423:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902a00. 00000080:00000010:1.1:1775764823.049433:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a600. 00010000:00000010:3.1:1775764823.049756:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7200. 00010000:00000010:3.1:1775764823.068202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7200. 00000080:00000010:3.1:1775764823.068212:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f920. 00010000:00000010:3.1:1775764823.144234:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798da00. 00010000:00000010:0.1:1775764823.158397:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207ce3100. 00010000:00000010:0.1:1775764823.158414:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798de00. 00000080:00000010:0.1:1775764823.176223:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472e60. 00000080:00000010:0.1:1775764823.176268:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d0000. 00010000:00000010:0.1:1775764823.190351:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0800. 00000080:00000010:1.1:1775764823.190924:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472e80. 00010000:00000010:3.1:1775764823.194197:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207ce3900. 00010000:00000010:3.1:1775764823.194220:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d400. 00010000:00000010:0.1:1775764823.240197:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c200. 00010000:00000010:1.1:1775764823.247411:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d600. 00010000:00000010:0.1:1775764823.248223:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1600. 00010000:00000010:3.1:1775764823.271347:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798de00. 00000080:00000010:3.1:1775764823.279580:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942074721e0. 00010000:00000010:3.1:1775764823.343026:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d400. 00010000:00000010:3.1:1775764823.347222:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d000. 00000080:00000010:1.1:1775764823.378304:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f6a0. 00010000:00000010:3.1:1775764823.414266:0:6168:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798ce00. 00000080:00000010:3.1:1775764823.425213:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2d80. 00010000:00000010:3.1:1775764823.430904:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6a00. 00010000:00000010:0.1:1775764823.431231:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207c67500. 00010000:00000010:0.1:1775764823.431238:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c200. 00010000:00000010:3.1:1775764823.498238:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798ce00. 00010000:00000010:3.1:1775764823.498246:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d000. 00000080:00000010:0.1:1775764823.511071:0:2392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db21a0. 00010000:00000010:0.1:1775764823.511092:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c200. 00000080:00000010:1.1:1775764823.547425:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472340. 00010000:00000010:3.1:1775764823.576631:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0c00. 00010000:00000010:3.1:1775764823.661233:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903800. 00010000:00000010:3.1:1775764823.661256:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d600. 00000080:00000010:0.1:1775764823.669036:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a820. 00010000:00000010:1.1:1775764823.699321:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0800. 00010000:00000010:3.1:1775764823.705512:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6a00. 00010000:00000010:3.1:1775764823.721241:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798de00. 00010000:00000010:3.1:1775764823.721248:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7200. 00010000:00000010:0.1:1775764823.723361:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078e9200. 00010000:00000010:0.1:1775764823.723397:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d200. 00000080:00000010:0.1:1775764823.744745:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2840. 00000080:00000010:3.1:1775764823.745195:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f880. 00000080:00000010:3.1:1775764823.745207:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472da0. 00010000:00000010:3.1:1775764823.749215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902a00. 00000080:00000010:3.1:1775764823.749240:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a0e0. 00000080:00000010:1.1:1775764823.749248:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d4a40. 00010000:00000010:3.1:1775764823.801034:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c200. 00010000:00000010:3.1:1775764823.810201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6a00. 00010000:00000010:3.1:1775764823.810256:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7200. 00010000:00000010:3.1:1775764823.841486:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d000. 00010000:00000010:1.1:1775764823.855318:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207ce3600. 00010000:00000010:1.1:1775764823.855329:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798ca00. 00000080:00000010:0.1:1775764823.882184:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756fe20. 00000080:00000010:0.1:1775764823.882205:0:14453:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d0840. 00010000:00000010:0.1:1775764823.901237:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903e00. 00000080:00000010:0.1:1775764823.901260:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a780. 00010000:00000010:0.1:1775764823.901276:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207c67200. 00010000:00000010:0.1:1775764823.901283:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7e00. 00010000:00000010:3.1:1775764823.910517:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798ce00. 00010000:00000010:0.1:1775764823.969234:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798ca00. 00000080:00000010:3.1:1775764823.978244:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2ce0. 00010000:00000010:3.1:1775764823.978264:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209d57c00. 00010000:00000010:3.1:1775764823.978268:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7200. 00000080:00000010:3.1:1775764824.026272:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756fca0. 00010000:00000010:3.1:1775764824.033344:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798ce00. 00010000:00000010:0.1:1775764824.034308:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7200. 00010000:00000010:3.1:1775764824.046292:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207976400. 00010000:00000010:3.1:1775764824.046301:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d000. 00000080:00000010:3.1:1775764824.063185:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2ec0. 00010000:00000010:3.1:1775764824.084186:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c200. 00000080:00000010:3.1:1775764824.103246:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2bc0. 00010000:00000010:1.1:1775764824.106351:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d200. 00010000:00000010:3.1:1775764824.120191:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6600. 00010000:00000010:3.1:1775764824.140709:0:6168:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078e9a00. 00010000:00000010:3.1:1775764824.140718:0:6168:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7a00. 00010000:00000010:3.1:1775764824.146344:0:2402:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994210812000. 00010000:00000010:3.1:1775764824.146356:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798dc00. 00010000:00000010:3.1:1775764824.146360:0:2402:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420788e900. 00010000:00000010:3.1:1775764824.146364:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6200. 00010000:00000010:3.1:1775764824.146368:0:2402:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078e9300. 00010000:00000010:3.1:1775764824.146371:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6e00. 00010000:00000010:3.1:1775764824.157648:0:2402:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942079b7100. 00010000:00000010:3.1:1775764824.157663:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7c00. 00010000:00000010:3.1:1775764824.157669:0:2402:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078e9e00. 00010000:00000010:3.1:1775764824.157674:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6400. 00010000:00000010:3.1:1775764824.157678:0:2402:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207926200. 00010000:00000010:3.1:1775764824.157682:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798cc00. 00010000:00000010:1.1:1775764824.158558:0:2392:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994205348b00. 00010000:00000010:1.1:1775764824.158567:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903c00. 00010000:00000010:3.1:1775764824.165384:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943900. 00010000:00000010:3.1:1775764824.165391:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902c00. 00010000:00000010:1.1:1775764824.184403:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d400. 00000080:00000010:3.1:1775764824.235289:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2960. 00010000:00000010:0.1:1775764824.239074:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798cc00. 00000080:00000010:0.1:1775764824.267373:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756fca0. 00010000:00000010:1.1:1775764824.289020:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0a00. 00010000:00000010:0.1:1775764824.296586:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1800. 00010000:00000010:1.1:1775764824.303265:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0800. 00010000:00000010:1.1:1775764824.303279:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7c00. 00000080:00000010:0.1:1775764824.313395:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472740. 00000080:00000010:0.1:1775764824.313411:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472340. 00010000:00000010:3.1:1775764824.344876:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1800. 00010000:00000010:3.1:1775764824.344883:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d600. 00010000:00000010:3.1:1775764824.350799:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903800. 00010000:00000010:3.1:1775764824.350817:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798dc00. 00000080:00000010:0.1:1775764824.382338:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f1a0. 00010000:00000010:1.1:1775764824.399257:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902a00. 00010000:00000010:0.1:1775764824.412907:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c200. 00010000:00000010:3.1:1775764824.418176:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7a00. 00010000:00000010:3.1:1775764824.438108:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6e00. 00010000:00000010:3.1:1775764824.438122:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d200. 00000080:00000010:3.1:1775764824.448198:0:1026:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756fc80. 00000080:00000010:3.1:1775764824.448226:0:1026:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2f00. 00000080:00000010:1.1:1775764824.458093:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f880. 00010000:00000010:3.1:1775764824.491210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7a00. 00010000:00000010:3.1:1775764824.518673:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903200. 00010000:00000010:3.1:1775764824.537211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798ce00. 00000080:00000010:3.1:1775764824.559372:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a440. 00010000:00000010:1.1:1775764824.601302:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7a00. 00010000:00000010:1.1:1775764824.610116:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d000. 00010000:00000010:3.1:1775764824.705209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6600. 00010000:00000010:3.1:1775764824.705229:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c200. 00010000:00000010:0.1:1775764824.709279:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d200. 00010000:00000010:0.1:1775764824.709291:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798ca00. 00000080:00000010:0.1:1775764824.717298:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756fd00. 00000080:00000010:0.1:1775764824.717319:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756fc80. 00010000:00000010:0.1:1775764824.742687:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798dc00. 00010000:00000010:3.1:1775764824.742996:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0600. 00000080:00000010:3.1:1775764824.774201:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2f40. 00010000:00000010:1.1:1775764824.788772:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6600. 00010000:00000010:3.1:1775764824.789199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6400. 00000080:00000010:3.1:1775764824.789205:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756fa20. 00010000:00000010:3.1:1775764824.855259:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207976500. 00010000:00000010:3.1:1775764824.855265:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798dc00. 00000080:00000010:3.1:1775764824.868206:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2040. 00000080:00000010:3.1:1775764824.868217:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756fcc0. 00010000:00000010:3.1:1775764824.887731:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0600. 00010000:00000010:0.1:1775764824.891438:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7a00. 00000080:00000010:3.1:1775764824.902175:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756fcc0. 00010000:00000010:3.1:1775764824.902194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798ca00. 00010000:00000010:3.1:1775764824.916180:0:1534:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6200. 00000080:00000010:0.1:1775764824.936363:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2360. 00010000:00000010:0.1:1775764824.936386:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d600. 00010000:00000010:1.1:1775764824.937210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c200. 00000080:00000010:3.1:1775764825.004213:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f620. 00000080:00000010:3.1:1775764825.017583:0:14453:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d0840. 00010000:00000010:3.1:1775764825.017599:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6400. 00010000:00000010:0.1:1775764825.020697:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d200. 00000080:00000010:0.1:1775764825.070642:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2720. 00010000:00000010:3.1:1775764825.101319:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903200. 00000080:00000010:1.1:1775764825.110192:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a620. 00010000:00000010:1.1:1775764825.110209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0a00. 00010000:00000010:1.1:1775764825.205034:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798ce00. 00000080:00000010:0.1:1775764825.224290:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472340. 00010000:00000010:0.1:1775764825.224310:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0600. 00010000:00000010:1.1:1775764825.235331:0:2400:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0800. 00000080:00000010:3.1:1775764825.281817:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2280. 00010000:00000010:1.1:1775764825.294287:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6200. 00000080:00000010:3.1:1775764825.339608:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2b00. 00000080:00000010:3.1:1775764825.339630:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472da0. 00010000:00000010:3.1:1775764825.370388:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d000. 00000080:00000010:3.1:1775764825.414563:0:14464:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472220. 00010000:00000010:3.1:1775764825.476741:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1c00. 00010000:00000010:3.1:1775764825.518255:0:1358:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798ce00. 00010000:00000010:3.1:1775764825.593743:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7e00. 00010000:00000010:1.1:1775764825.633761:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c200. 00000080:00000010:3.1:1775764825.676832:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2180. 00010000:00000010:0.1:1775764825.691300:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7e00. 00000080:00000010:0.1:1775764825.708368:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a5a0. 00000080:00000010:0.1:1775764825.708396:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756fa20. 00010000:00000010:0.1:1775764825.708407:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d200. 00010000:00000010:0.1:1775764825.708412:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942079b7400. 00010000:00000010:0.1:1775764825.708416:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7000. 00010000:00000010:0.1:1775764825.708419:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6800. 00010000:00000010:3.1:1775764825.745496:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81a00. 00000080:00000010:0.1:1775764825.745785:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756fa00. 00010000:00000010:3.1:1775764825.777601:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798da00. 00010000:00000010:3.1:1775764825.806275:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798ca00. 00010000:00000010:0.1:1775764825.823271:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798dc00. 00000080:00000010:3.1:1775764825.829260:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2680. 00010000:00000010:1.1:1775764825.848780:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1a00. 00010000:00000010:0.1:1775764825.876750:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6800. 00010000:00000010:0.1:1775764825.876761:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7000. 00010000:00000010:0.1:1775764825.876766:0:2392:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ada00. 00010000:00000010:0.1:1775764825.876770:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0400. 00000080:00000010:1.1:1775764825.884061:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f220. 00000080:00000010:1.1:1775764825.884088:0:14453:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d4200. 00010000:00000010:3.1:1775764825.906054:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1c00. 00010000:00000010:1.1:1775764825.952709:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798ca00. 00010000:00000010:1.1:1775764825.952726:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798da00. 00010000:00000010:3.1:1775764826.034448:0:6168:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7200. 00010000:00000010:1.1:1775764826.053254:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6200. 00010000:00000010:3.1:1775764826.053331:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81c00. 00010000:00000010:3.1:1775764826.137168:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0e00. 00010000:00000010:3.1:1775764826.168259:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c200. 00010000:00000010:3.1:1775764826.172224:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1a00. 00010000:00000010:3.1:1775764826.219395:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d000. 00010000:00000010:1.1:1775764826.252022:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c200. 00010000:00000010:0.1:1775764826.266372:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902600. 00010000:00000010:3.1:1775764826.285231:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798da00. 00010000:00000010:3.1:1775764826.316182:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d000. 00010000:00000010:3.1:1775764826.335249:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0200. 00010000:00000010:3.1:1775764826.361229:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798da00. 00010000:00000010:3.1:1775764826.361239:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1600. 00010000:00000010:3.1:1775764826.368271:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d000. 00010000:00000010:1.1:1775764826.395414:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1600. 00010000:00000010:3.1:1775764826.423533:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da200. 00010000:00000010:3.1:1775764826.453819:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0000. 00010000:00000010:1.1:1775764826.456244:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6600. 00010000:00000010:3.1:1775764826.457105:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798da00. 00010000:00000010:3.1:1775764826.462869:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6e00. 00010000:00000010:3.1:1775764826.476828:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0c00. 00010000:00000010:3.1:1775764826.485272:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079cac00. 00010000:00000010:3.1:1775764826.512262:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079ca200. 00010000:00000010:0.1:1775764826.529500:0:2400:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6e00. 00010000:00000010:1.1:1775764826.542488:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0a00. 00010000:00000010:3.1:1775764826.567627:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7c00. 00010000:00000010:3.1:1775764826.583419:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1800. 00010000:00000010:3.1:1775764826.587231:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0400. 00010000:00000010:3.1:1775764826.626248:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207976700. 00010000:00000010:3.1:1775764826.626264:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079cb000. 00010000:00000010:3.1:1775764826.645288:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1a00. 00010000:00000010:0.1:1775764826.665386:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1800. 00010000:00000010:3.1:1775764826.670377:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079ca800. 00010000:00000010:0.1:1775764826.693114:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079ca800. 00010000:00000010:0.1:1775764826.710651:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1800. 00010000:00000010:3.1:1775764826.732356:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902600. 00010000:00000010:1.1:1775764826.755782:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1800. 00000080:00000010:1.1:1775764826.759524:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6aba0. 00000080:00000010:1.1:1775764826.759536:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472620. 00000080:00000010:1.1:1775764826.759541:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db29a0. 00000080:00000010:1.1:1775764826.759546:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db27a0. 00010000:00000010:1.1:1775764826.759552:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c200. 00010000:00000010:3.1:1775764826.834196:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943b00. 00010000:00000010:3.1:1775764826.841269:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902c00. 00010000:00000010:3.1:1775764826.852612:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207976800. 00010000:00000010:3.1:1775764826.852619:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079caa00. 00010000:00000010:3.1:1775764826.887293:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903c00. 00010000:00000010:0.1:1775764826.898196:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902a00. 00010000:00000010:3.1:1775764826.935098:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079ca200. 00000080:00000010:3.1:1775764826.954733:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db24e0. 00010000:00000010:3.1:1775764826.987337:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903200. 00010000:00000010:1.1:1775764827.011228:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798ce00. 00010000:00000010:3.1:1775764827.027376:0:31:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7000. 00010000:00000010:1.1:1775764827.046388:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6000. 00000080:00000010:3.1:1775764827.059747:0:2402:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f400. 00010000:00000010:3.1:1775764827.078275:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207976800. 00010000:00000010:3.1:1775764827.078282:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902800. 00010000:00000010:3.1:1775764827.081296:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1a00. 00010000:00000010:3.1:1775764827.120254:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c200. 00000080:00000010:0.1:1775764827.140765:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6ad80. 00010000:00000010:3.1:1775764827.143478:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793d000. 00000080:00000010:3.1:1775764827.143527:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db23e0. 00010000:00000010:3.1:1775764827.143533:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d000. 00010000:00000010:1.1:1775764827.148325:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1200. 00010000:00000010:3.1:1775764827.168290:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0600. 00010000:00000010:3.1:1775764827.194428:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1c00. 00010000:00000010:3.1:1775764827.206207:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209d57700. 00010000:00000010:3.1:1775764827.206214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7000. 00010000:00000010:1.1:1775764827.264831:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1800. 00000080:00000010:3.1:1775764827.273725:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472d80. 00010000:00000010:3.1:1775764827.315242:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793c600. 00000080:00000010:1.1:1775764827.346271:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2780. 00010000:00000010:3.1:1775764827.346999:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7000. 00010000:00000010:0.1:1775764827.350508:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903c00. 00010000:00000010:3.1:1775764827.372177:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0400. 00010000:00000010:0.1:1775764827.386259:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6600. 00010000:00000010:3.1:1775764827.415879:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793d800. 00010000:00000010:0.1:1775764827.457341:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0400. 00010000:00000010:3.1:1775764827.466451:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793d000. 00000080:00000010:1.1:1775764827.478375:0:13:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f1a0. 00010000:00000010:3.1:1775764827.493344:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902400. 00000080:00000010:0.1:1775764827.511879:0:2392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756fca0. 00010000:00000010:3.1:1775764827.544563:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0600. 00010000:00000010:0.1:1775764827.553199:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1200. 00000080:00000010:0.1:1775764827.553205:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472bc0. 00010000:00000010:3.1:1775764827.606184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7000. 00000080:00000010:1.1:1775764827.642771:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472020. 00010000:00000010:0.1:1775764827.652199:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903800. 00000080:00000010:1.1:1775764827.659394:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2fe0. 00000080:00000010:1.1:1775764827.669289:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f540. 00000080:00000010:1.1:1775764827.674340:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2c40. 00000080:00000010:1.1:1775764827.690194:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472760. 00000080:00000010:1.1:1775764827.702267:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2fa0. 00000080:00000010:1.1:1775764827.710202:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756fc80. 00010000:00000010:3.1:1775764827.727266:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994203fb0600. 00000080:00000010:1.1:1775764827.729084:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472da0. 00000080:00000010:1.1:1775764827.755931:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2820. 00000080:00000010:1.1:1775764827.756026:0:20:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756fc60. 00010000:00000010:1.1:1775764827.759845:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0c00. 00010000:00000010:1.1:1775764827.759861:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078add00. 00010000:00000010:1.1:1775764827.759865:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798da00. 00010000:00000010:1.1:1775764827.759868:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0600. 00000080:00000010:1.1:1775764827.772345:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a880. 00000080:00000010:1.1:1775764827.772392:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d0000. 00010000:00000010:3.1:1775764827.807200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994203fb0200. 00010000:00000010:0.1:1775764827.838228:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079cac00. 00010000:00000010:0.1:1775764827.838235:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209d57c00. 00010000:00000010:0.1:1775764827.838239:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7c00. 00010000:00000010:3.1:1775764827.861352:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7000. 00000080:00000010:3.1:1775764827.888092:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2500. 00010000:00000010:1.1:1775764827.891323:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7000. 00010000:00000010:3.1:1775764827.909242:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0a00. 00010000:00000010:3.1:1775764827.909250:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994203fb0000. 00010000:00000010:1.1:1775764827.929013:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079cbc00. 00010000:00000010:3.1:1775764827.948270:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6000. 00010000:00000010:3.1:1775764827.960203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994203fb0200. 00010000:00000010:3.1:1775764827.978523:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207535e00. 00010000:00000010:1.1:1775764828.000219:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902800. 00000080:00000010:1.1:1775764828.015300:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472620. 00010000:00000010:3.1:1775764828.021233:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207534800. 00010000:00000010:3.1:1775764828.021241:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902400. 00010000:00000010:1.1:1775764828.061541:0:6159:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994218f6f000. 00010000:00000010:3.1:1775764828.062196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902c00. 00010000:00000010:3.1:1775764828.138250:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971a00. 00010000:00000010:3.1:1775764828.138260:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6200. 00010000:00000010:3.1:1775764828.187826:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903200. 00010000:00000010:3.1:1775764828.194768:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971400. 00010000:00000010:3.1:1775764828.223355:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970400. 00010000:00000010:3.1:1775764828.251233:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970800. 00010000:00000010:3.1:1775764828.254310:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970a00. 00010000:00000010:1.1:1775764828.285201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971c00. 00010000:00000010:3.1:1775764828.285287:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902a00. 00010000:00000010:3.1:1775764828.288318:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079ca000. 00010000:00000010:3.1:1775764828.324295:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6200. 00010000:00000010:3.1:1775764828.375215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903800. 00010000:00000010:3.1:1775764828.395188:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970000. 00010000:00000010:3.1:1775764828.407202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6200. 00010000:00000010:3.1:1775764828.447991:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0600. 00000080:00000010:3.1:1775764828.470293:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2a40. 00010000:00000010:3.1:1775764828.509194:0:1026:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6200. 00000080:00000010:0.1:1775764828.587863:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942074727a0. 00000080:00000010:3.1:1775764828.621275:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2420. 00010000:00000010:3.1:1775764828.698247:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7c00. 00010000:00000010:0.1:1775764828.698899:0:2392:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207976400. 00010000:00000010:0.1:1775764828.698907:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7e00. 00010000:00000010:0.1:1775764828.698911:0:2392:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207ce3e00. 00010000:00000010:0.1:1775764828.698914:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0e00. 00010000:00000010:0.1:1775764828.698916:0:2392:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207976d00. 00010000:00000010:0.1:1775764828.698919:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798cc00. 00010000:00000010:0.1:1775764828.698922:0:2392:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078e9d00. 00010000:00000010:0.1:1775764828.698926:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7800. 00010000:00000010:0.1:1775764828.699396:0:2392:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207976600. 00010000:00000010:0.1:1775764828.699403:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d400. 00010000:00000010:0.1:1775764828.703214:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207931000. 00010000:00000010:0.1:1775764828.703232:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798ce00. 00010000:00000010:0.1:1775764828.703237:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ad300. 00010000:00000010:0.1:1775764828.703240:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7600. 00010000:00000010:3.1:1775764828.762941:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971600. 00010000:00000010:0.1:1775764828.827883:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971e00. 00010000:00000010:3.1:1775764828.849370:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6800. 00010000:00000010:3.1:1775764828.854205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6600. 00010000:00000010:3.1:1775764828.939334:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209d57c00. 00010000:00000010:3.1:1775764828.939348:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970200. 00010000:00000010:3.1:1775764829.009384:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0400. 00010000:00000010:3.1:1775764829.023405:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970200. 00010000:00000010:3.1:1775764829.058401:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6600. 00000080:00000010:3.1:1775764829.097342:0:1026:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f800. 00010000:00000010:3.1:1775764829.130313:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6600. 00010000:00000010:3.1:1775764829.157204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902200. 00010000:00000010:3.1:1775764829.173383:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971600. 00010000:00000010:3.1:1775764829.198237:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970000. 00010000:00000010:3.1:1775764829.218750:0:14452:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970600. 00010000:00000010:0.1:1775764829.256200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971400. 00010000:00000010:3.1:1775764829.268209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970e00. 00010000:00000010:3.1:1775764829.272217:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971a00. 00010000:00000010:0.1:1775764829.276268:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6800. 00010000:00000010:1.1:1775764829.301452:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903e00. 00000080:00000010:1.1:1775764829.301467:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a900. 00010000:00000010:3.1:1775764829.383251:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ad300. 00010000:00000010:3.1:1775764829.383268:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970600. 00010000:00000010:3.1:1775764829.399210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0800. 00010000:00000010:3.1:1775764829.440376:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ade00. 00010000:00000010:3.1:1775764829.440395:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970400. 00010000:00000010:3.1:1775764829.483292:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0600. 00010000:00000010:3.1:1775764829.547276:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971600. 00010000:00000010:3.1:1775764829.584200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970000. 00000080:00000010:3.1:1775764829.631216:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2b20. 00010000:00000010:0.1:1775764829.687220:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6e00. 00010000:00000010:3.1:1775764829.695329:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903e00. 00010000:00000010:3.1:1775764829.755143:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6800. 00010000:00000010:3.1:1775764829.824240:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6800. 00000080:00000010:3.1:1775764829.836428:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2800. 00000080:00000010:3.1:1775764829.881602:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f800. 00010000:00000010:3.1:1775764829.891452:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970600. 00010000:00000010:3.1:1775764829.891462:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966100. 00010000:00000010:3.1:1775764829.891466:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0800. 00010000:00000010:3.1:1775764829.891469:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6600. 00000080:00000010:1.1:1775764829.918542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f7a0. 00010000:00000010:3.1:1775764829.919284:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6800. 00010000:00000010:3.1:1775764829.945117:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902800. 00010000:00000010:0.1:1775764829.953115:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971a00. 00010000:00000010:3.1:1775764829.957402:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970600. 00000080:00000010:3.1:1775764829.957412:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2760. 00000080:00000010:0.1:1775764829.969290:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f320. 00010000:00000010:0.1:1775764829.969299:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970400. 00010000:00000010:0.1:1775764829.969305:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943b00. 00010000:00000010:0.1:1775764829.969308:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902200. 00010000:00000010:0.1:1775764829.969312:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970e00. 00000080:00000010:0.1:1775764829.981188:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6ad60. 00000080:00000010:0.1:1775764829.981211:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942276298c0. 00000080:00000010:1.1:1775764830.005413:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472760. 00010000:00000010:0.1:1775764830.008369:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7000. 00010000:00000010:1.1:1775764830.017897:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903e00. 00000080:00000010:1.1:1775764830.017907:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a760. 00010000:00000010:1.1:1775764830.094046:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903e00. 00010000:00000010:1.1:1775764830.096791:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902200. 00010000:00000010:3.1:1775764830.122272:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1c00. 00010000:00000010:3.1:1775764830.136515:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6600. 00010000:00000010:0.1:1775764830.185372:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971600. 00010000:00000010:3.1:1775764830.215415:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6600. 00010000:00000010:3.1:1775764830.273721:0:6169:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903e00. 00010000:00000010:3.1:1775764830.362359:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7c00. 00010000:00000010:3.1:1775764830.362367:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970000. 00010000:00000010:3.1:1775764830.395206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971600. 00010000:00000010:3.1:1775764830.411592:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970400. 00000080:00000010:3.1:1775764830.438228:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2a60. 00010000:00000010:3.1:1775764830.463314:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971a00. 00000080:00000010:3.1:1775764830.471468:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a260. 00000080:00000010:3.1:1775764830.512220:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2f60. 00000080:00000010:3.1:1775764830.512241:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db22a0. 00000080:00000010:0.1:1775764830.537652:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2880. 00000080:00000010:3.1:1775764830.573206:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f2c0. 00000080:00000010:3.1:1775764830.577224:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff99422762b9c0. 00010000:00000010:1.1:1775764830.604216:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7c00. 00000080:00000010:1.1:1775764830.604223:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f800. 00010000:00000010:0.1:1775764830.693911:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970600. 00010000:00000010:3.1:1775764830.709237:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a6a00. 00010000:00000010:1.1:1775764830.761344:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898400. 00010000:00000010:3.1:1775764830.801690:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6c00. 00010000:00000010:3.1:1775764830.833207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971600. 00010000:00000010:1.1:1775764830.836550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902200. 00000080:00000010:3.1:1775764830.840291:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f2c0. 00010000:00000010:3.1:1775764830.873206:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7800. 00010000:00000010:3.1:1775764830.881324:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970000. 00010000:00000010:3.1:1775764830.917209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079ca400. 00010000:00000010:3.1:1775764830.926765:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903e00. 00010000:00000010:3.1:1775764830.939196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7c00. 00010000:00000010:3.1:1775764830.959295:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902c00. 00010000:00000010:3.1:1775764830.971349:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970400. 00010000:00000010:0.1:1775764830.979213:0:2401:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209d57600. 00010000:00000010:0.1:1775764830.979229:0:2401:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075ca800. 00010000:00000010:3.1:1775764830.983302:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207c67900. 00010000:00000010:3.1:1775764830.983309:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da600. 00010000:00000010:3.1:1775764830.999186:0:14464:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902200. 00010000:00000010:3.1:1775764831.011246:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7c00. 00010000:00000010:0.1:1775764831.020071:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970000. 00010000:00000010:0.1:1775764831.076891:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207c67600. 00010000:00000010:0.1:1775764831.076904:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cbca00. 00010000:00000010:3.1:1775764831.077302:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971600. 00010000:00000010:3.1:1775764831.099212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970600. 00010000:00000010:3.1:1775764831.135783:0:2402:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420788ef00. 00010000:00000010:3.1:1775764831.135797:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079daa00. 00010000:00000010:3.1:1775764831.148412:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971600. 00010000:00000010:3.1:1775764831.158182:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7800. 00010000:00000010:3.1:1775764831.175289:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971600. 00010000:00000010:3.1:1775764831.186340:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209d57900. 00010000:00000010:3.1:1775764831.186355:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789c000. 00010000:00000010:0.1:1775764831.218185:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902200. 00010000:00000010:3.1:1775764831.236201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895e00. 00010000:00000010:1.1:1775764831.241550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903c00. 00010000:00000010:3.1:1775764831.270202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903c00. 00010000:00000010:3.1:1775764831.286614:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895200. 00010000:00000010:1.1:1775764831.314320:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209d57200. 00010000:00000010:1.1:1775764831.314335:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ce1200. 00010000:00000010:3.1:1775764831.327659:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994211bdbf00. 00010000:00000010:3.1:1775764831.327667:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00010000:00000010:1.1:1775764831.336212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7800. 00010000:00000010:3.1:1775764831.369187:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7e00. 00010000:00000010:1.1:1775764831.389566:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6400. 00010000:00000010:0.1:1775764831.403697:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894400. 00010000:00000010:0.1:1775764831.469641:0:6169:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994201482600. 00010000:00000010:0.1:1775764831.478209:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209d57800. 00010000:00000010:0.1:1775764831.478227:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994218f6ee00. 00010000:00000010:3.1:1775764831.533329:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6c00. 00010000:00000010:3.1:1775764831.538234:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6400. 00010000:00000010:0.1:1775764831.568188:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895400. 00010000:00000010:1.1:1775764831.580111:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6c00. 00010000:00000010:3.1:1775764831.586203:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209d57000. 00010000:00000010:3.1:1775764831.586210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994218f6e800. 00010000:00000010:0.1:1775764831.614203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7800. 00010000:00000010:3.1:1775764831.639320:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895800. 00010000:00000010:3.1:1775764831.698429:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075cac00. 00010000:00000010:3.1:1775764831.719343:0:14464:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075ca800. 00010000:00000010:0.1:1775764831.737081:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894600. 00010000:00000010:3.1:1775764831.741257:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994208aff200. 00010000:00000010:3.1:1775764831.741265:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb3800. 00010000:00000010:0.1:1775764831.799655:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a6a00. 00010000:00000010:0.1:1775764831.823335:0:6159:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075cac00. 00000080:00000010:0.1:1775764831.823395:0:6159:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942074729c0. 00000080:00000010:0.1:1775764831.823404:0:6159:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a080. 00010000:00000010:0.1:1775764831.823410:0:6159:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970e00. 00010000:00000010:3.1:1775764831.845446:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966a00. 00010000:00000010:3.1:1775764831.845457:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894400. 00010000:00000010:3.1:1775764831.901399:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895200. 00000080:00000010:3.1:1775764831.977362:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472620. 00010000:00000010:1.1:1775764831.996475:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895400. 00010000:00000010:1.1:1775764832.000281:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7600. 00000080:00000010:1.1:1775764832.089329:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6ad80. 00010000:00000010:1.1:1775764832.109324:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a6200. 00010000:00000010:3.1:1775764832.129262:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075ca800. 00010000:00000010:3.1:1775764832.129269:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895e00. 00010000:00000010:1.1:1775764832.159348:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075cac00. 00000080:00000010:1.1:1775764832.159363:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472e60. 00010000:00000010:3.1:1775764832.183094:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895400. 00010000:00000010:1.1:1775764832.231292:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917a00. 00000080:00000010:3.1:1775764832.233197:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f780. 00000080:00000010:3.1:1775764832.255272:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f620. 00000080:00000010:1.1:1775764832.266092:0:2392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2b20. 00010000:00000010:0.1:1775764832.299198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895e00. 00010000:00000010:1.1:1775764832.316215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895400. 00010000:00000010:1.1:1775764832.329210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a6400. 00000080:00000010:1.1:1775764832.337315:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f000. 00010000:00000010:3.1:1775764832.372247:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a6400. 00010000:00000010:3.1:1775764832.372262:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895e00. 00000080:00000010:3.1:1775764832.440179:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2aa0. 00010000:00000010:3.1:1775764832.455209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970600. 00000080:00000010:3.1:1775764832.455273:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db22e0. 00000080:00000010:3.1:1775764832.455279:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a5a0. 00010000:00000010:3.1:1775764832.455284:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a6200. 00010000:00000010:0.1:1775764832.455342:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917000. 00000080:00000010:0.1:1775764832.517312:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db21e0. 00000080:00000010:0.1:1775764832.517323:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2b60. 00010000:00000010:0.1:1775764832.517330:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a6800. 00010000:00000010:3.1:1775764832.521217:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075dae00. 00010000:00000010:3.1:1775764832.537194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916600. 00010000:00000010:3.1:1775764832.583233:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916c00. 00000080:00000010:3.1:1775764832.595787:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2f80. 00010000:00000010:0.1:1775764832.609396:0:14461:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916600. 00010000:00000010:0.1:1775764832.642799:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075cac00. 00010000:00000010:1.1:1775764832.655252:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420b14ac00. 00010000:00000010:3.1:1775764832.660198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420b14a400. 00000080:00000010:0.1:1775764832.683452:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f620. 00010000:00000010:0.1:1775764832.711409:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2a00. 00010000:00000010:0.1:1775764832.751244:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ad700. 00010000:00000010:0.1:1775764832.751252:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903200. 00010000:00000010:3.1:1775764832.762257:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb3600. 00010000:00000010:1.1:1775764832.810337:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917200. 00010000:00000010:1.1:1775764833.047199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075ca800. 00010000:00000010:3.1:1775764833.047234:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209d57f00. 00010000:00000010:3.1:1775764833.047241:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2600. 00000080:00000010:0.1:1775764833.172511:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2500. 00010000:00000010:1.1:1775764833.181320:0:6158:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cbcc00. 00010000:00000010:3.1:1775764833.192389:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916400. 00010000:00000010:0.1:1775764833.268202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cbd800. 00000080:00000010:0.1:1775764833.268213:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472e80. 00010000:00000010:3.1:1775764833.292541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cbca00. 00010000:00000010:3.1:1775764833.296374:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2800. 00010000:00000010:0.1:1775764833.404673:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2600. 00010000:00000010:3.1:1775764833.440271:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb3e00. 00010000:00000010:3.1:1775764833.452196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb3600. 00010000:00000010:3.1:1775764833.452205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916800. 00010000:00000010:3.1:1775764833.518209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb3600. 00000080:00000010:3.1:1775764833.518226:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2260. 00000080:00000010:1.1:1775764833.528312:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f800. 00010000:00000010:3.1:1775764833.574216:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2a00. 00010000:00000010:1.1:1775764833.574375:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917200. 00010000:00000010:3.1:1775764833.650193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916c00. 00010000:00000010:3.1:1775764833.680193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2a00. 00000080:00000010:3.1:1775764833.680201:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2fa0. 00010000:00000010:1.1:1775764833.733300:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420694d400. 00010000:00000010:3.1:1775764833.767067:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916600. 00010000:00000010:3.1:1775764833.774198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2800. 00000080:00000010:0.1:1775764833.785211:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2660. 00000080:00000010:1.1:1775764833.790308:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6aa40. 00000080:00000010:1.1:1775764833.794311:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a260. 00000080:00000010:1.1:1775764833.800005:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2b80. 00010000:00000010:3.1:1775764833.800461:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2800. 00000080:00000010:1.1:1775764833.808281:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6aba0. 00000080:00000010:1.1:1775764833.811405:0:6158:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f860. 00010000:00000010:3.1:1775764833.820264:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb3600. 00000080:00000010:3.1:1775764833.835179:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2520. 00010000:00000010:3.1:1775764833.843709:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2a00. 00000080:00000010:3.1:1775764833.847656:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756fa00. 00010000:00000010:1.1:1775764833.853420:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd400. 00000080:00000010:1.1:1775764833.853427:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a9c0. 00000080:00000010:0.1:1775764833.858241:0:12:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6ad80. 00010000:00000010:0.1:1775764833.858264:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cbce00. 00010000:00000010:0.1:1775764833.858269:0:12:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209d57f00. 00010000:00000010:0.1:1775764833.858273:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075cbc00. 00010000:00000010:0.1:1775764833.858276:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916c00. 00000080:00000010:3.1:1775764833.858331:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a900. 00000080:00000010:3.1:1775764833.868766:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db22c0. 00010000:00000010:3.1:1775764833.891200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2800. 00010000:00000010:3.1:1775764833.900178:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917800. 00010000:00000010:0.1:1775764833.900437:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2a00. 00000080:00000010:0.1:1775764833.916354:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2820. 00010000:00000010:3.1:1775764833.930990:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0400. 00000080:00000010:0.1:1775764833.941491:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2560. 00010000:00000010:3.1:1775764833.947393:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1000. 00000080:00000010:0.1:1775764833.959133:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2f60. 00010000:00000010:1.1:1775764833.964362:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0a00. 00010000:00000010:0.1:1775764833.970297:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0c00. 00000080:00000010:0.1:1775764833.970303:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942074729c0. 00010000:00000010:3.1:1775764834.032182:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2800. 00010000:00000010:3.1:1775764834.032221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2a00. 00010000:00000010:3.1:1775764834.032224:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1600. 00010000:00000010:3.1:1775764834.045203:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209d57800. 00010000:00000010:3.1:1775764834.045220:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb3e00. 00000080:00000010:1.1:1775764834.054176:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db22a0. 00000080:00000010:1.1:1775764834.054187:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472e60. 00010000:00000010:3.1:1775764834.080710:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942069a5400. 00010000:00000010:3.1:1775764834.080735:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd600. 00000080:00000010:3.1:1775764834.086913:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756fc80. 00000080:00000010:3.1:1775764834.086957:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2c40. 00010000:00000010:3.1:1775764834.086964:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2600. 00010000:00000010:3.1:1775764834.086969:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994206989500. 00010000:00000010:3.1:1775764834.086973:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2200. 00010000:00000010:3.1:1775764834.086976:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fca00. 00000080:00000010:0.1:1775764834.107366:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db25a0. 00010000:00000010:0.1:1775764834.128463:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc600. 00010000:00000010:1.1:1775764834.133823:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0200. 00010000:00000010:3.1:1775764834.143243:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb3600. 00010000:00000010:1.1:1775764834.254892:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2600. 00010000:00000010:3.1:1775764834.265481:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943700. 00010000:00000010:3.1:1775764834.265491:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb3e00. 00010000:00000010:3.1:1775764834.269227:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916600. 00000080:00000010:3.1:1775764834.269244:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756fac0. 00010000:00000010:3.1:1775764834.354546:0:2402:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994210812000. 00010000:00000010:3.1:1775764834.354558:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420694dc00. 00000080:00000010:1.1:1775764834.358354:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2900. 00000080:00000010:1.1:1775764834.358371:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2160. 00010000:00000010:3.1:1775764834.396962:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ad700. 00010000:00000010:3.1:1775764834.396969:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1800. 00010000:00000010:3.1:1775764834.418724:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc600. 00010000:00000010:3.1:1775764834.418730:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2200. 00010000:00000010:1.1:1775764834.444229:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0a00. 00000080:00000010:0.1:1775764834.444354:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472e60. 00010000:00000010:3.1:1775764834.458194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2800. 00010000:00000010:0.1:1775764834.501228:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1000. 00010000:00000010:3.1:1775764834.529440:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ad700. 00010000:00000010:3.1:1775764834.529454:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1a00. 00010000:00000010:3.1:1775764834.614793:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0200. 00010000:00000010:0.1:1775764834.624206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6600. 00000080:00000010:1.1:1775764834.630288:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942074720c0. 00010000:00000010:1.1:1775764834.647027:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd400. 00010000:00000010:3.1:1775764834.722221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc600. 00000080:00000010:3.1:1775764834.722240:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a900. 00010000:00000010:1.1:1775764834.722740:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2200. 00010000:00000010:3.1:1775764834.752269:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7c00. 00010000:00000010:0.1:1775764834.763881:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2200. 00010000:00000010:0.1:1775764834.793235:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1a00. 00010000:00000010:1.1:1775764834.816519:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2200. 00010000:00000010:3.1:1775764834.844301:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7e00. 00010000:00000010:0.1:1775764834.919190:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0600. 00010000:00000010:3.1:1775764834.946227:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fde00. 00010000:00000010:3.1:1775764834.989789:0:2403:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209d57c00. 00010000:00000010:3.1:1775764834.989801:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6e00. 00010000:00000010:3.1:1775764834.989804:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0200. 00010000:00000010:0.1:1775764834.991411:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc000. 00010000:00000010:3.1:1775764835.030228:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970400. 00010000:00000010:3.1:1775764835.088299:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994209db3000. 00010000:00000010:3.1:1775764835.131976:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994209db3000. 00010000:00000010:3.1:1775764835.137275:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1000. 00010000:00000010:3.1:1775764835.141203:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943800. 00010000:00000010:3.1:1775764835.141210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902c00. 00010000:00000010:3.1:1775764835.161205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc000. 00010000:00000010:3.1:1775764835.173277:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ce1800. 00010000:00000010:3.1:1775764835.181261:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966800. 00010000:00000010:3.1:1775764835.181268:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970000. 00010000:00000010:3.1:1775764835.199471:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ce0800. 00010000:00000010:3.1:1775764835.216861:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ce1600. 00010000:00000010:3.1:1775764835.231744:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994205348600. 00010000:00000010:3.1:1775764835.231761:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420b14a000. 00010000:00000010:0.1:1775764835.234485:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ce0800. 00010000:00000010:3.1:1775764835.256242:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966000. 00010000:00000010:3.1:1775764835.256258:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971600. 00010000:00000010:3.1:1775764835.277613:0:14473:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ce0200. 00010000:00000010:0.1:1775764835.322408:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ce1600. 00010000:00000010:3.1:1775764835.322669:0:32:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943a00. 00010000:00000010:3.1:1775764835.322674:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994201483e00. 00010000:00000010:3.1:1775764835.347839:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ce0200. 00010000:00000010:3.1:1775764835.356371:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966e00. 00010000:00000010:3.1:1775764835.356379:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894a00. 00010000:00000010:3.1:1775764835.401261:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966f00. 00010000:00000010:3.1:1775764835.401268:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895600. 00010000:00000010:0.1:1775764835.414420:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ce1600. 00010000:00000010:0.1:1775764835.427212:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994219da2300. 00010000:00000010:0.1:1775764835.427225:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970e00. 00010000:00000010:3.1:1775764835.430313:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917800. 00010000:00000010:0.1:1775764835.464686:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00010000:00000010:3.1:1775764835.474815:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943b00. 00010000:00000010:3.1:1775764835.474822:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075dbe00. 00010000:00000010:3.1:1775764835.486196:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209d57900. 00010000:00000010:3.1:1775764835.486203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7e00. 00010000:00000010:3.1:1775764835.502429:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898000. 00010000:00000010:1.1:1775764835.531300:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1800. 00010000:00000010:3.1:1775764835.548422:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1000. 00010000:00000010:3.1:1775764835.583073:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898400. 00010000:00000010:3.1:1775764835.593208:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc600. 00010000:00000010:3.1:1775764835.593226:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966c00. 00010000:00000010:3.1:1775764835.593230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895800. 00010000:00000010:0.1:1775764835.628180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899e00. 00010000:00000010:3.1:1775764835.648500:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899c00. 00010000:00000010:3.1:1775764835.648578:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966700. 00010000:00000010:3.1:1775764835.648583:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894200. 00010000:00000010:1.1:1775764835.666460:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899000. 00010000:00000010:3.1:1775764835.696359:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0c00. 00010000:00000010:3.1:1775764835.701195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894600. 00010000:00000010:3.1:1775764835.717302:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898c00. 00010000:00000010:0.1:1775764835.718234:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc800. 00010000:00000010:3.1:1775764835.726233:0:13:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ada00. 00010000:00000010:3.1:1775764835.726242:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6a00. 00010000:00000010:1.1:1775764835.731590:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899a00. 00000080:00000010:0.1:1775764835.732180:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2960. 00010000:00000010:3.1:1775764835.788127:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916200. 00010000:00000010:3.1:1775764835.811605:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cbd800. 00000080:00000010:1.1:1775764835.827183:0:2392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a760. 00000080:00000010:3.1:1775764835.873381:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a280. 00000080:00000010:3.1:1775764835.878322:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2880. 00000080:00000010:0.1:1775764835.891196:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6ad80. 00000080:00000010:0.1:1775764835.899510:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f7a0. 00000080:00000010:3.1:1775764835.934269:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db28a0. 00000080:00000010:1.1:1775764835.945328:0:20:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2e40. 00000080:00000010:1.1:1775764835.945348:0:20:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db20e0. 00010000:00000010:1.1:1775764835.945373:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899600. 00010000:00000010:1.1:1775764835.945379:0:20:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209d57d00. 00010000:00000010:1.1:1775764835.945383:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7800. 00010000:00000010:1.1:1775764835.945386:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2600. 00000080:00000010:3.1:1775764835.953581:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2040. 00000080:00000010:0.1:1775764835.954659:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756fca0. 00000080:00000010:3.1:1775764835.967312:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d6300. 00010000:00000010:3.1:1775764836.015364:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb3800. 00000080:00000010:1.1:1775764836.070551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a5c0. 00000080:00000010:3.1:1775764836.073412:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420130b500. 00000080:00000010:3.1:1775764836.077348:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2be0. 00010000:00000010:0.1:1775764836.082304:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cbca00. 00000080:00000010:3.1:1775764836.084648:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472e60. 00000080:00000010:3.1:1775764836.106207:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2680. 00000080:00000010:3.1:1775764836.106250:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942074729c0. 00000080:00000010:3.1:1775764836.106255:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2dc0. 00010000:00000010:3.1:1775764836.106259:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc200. 00010000:00000010:3.1:1775764836.106263:0:32:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966a00. 00010000:00000010:3.1:1775764836.106267:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895200. 00010000:00000010:3.1:1775764836.106285:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917a00. 00000080:00000010:3.1:1775764836.113031:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6ad60. 00010000:00000010:0.1:1775764836.122214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cbce00. 00010000:00000010:0.1:1775764836.122227:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ad300. 00010000:00000010:0.1:1775764836.122247:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075dae00. 00000080:00000010:1.1:1775764836.122696:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227606b40. 00000080:00000010:0.1:1775764836.130955:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f000. 00000080:00000010:0.1:1775764836.130976:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994218994200. 00010000:00000010:3.1:1775764836.133512:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb3200. 00010000:00000010:3.1:1775764836.159344:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898800. 00010000:00000010:3.1:1775764836.221936:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899c00. 00010000:00000010:3.1:1775764836.221942:0:32:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943900. 00010000:00000010:3.1:1775764836.221946:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ce0800. 00000080:00000010:3.1:1775764836.235371:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420130b820. 00010000:00000010:1.1:1775764836.235595:0:6159:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cbcc00. 00010000:00000010:3.1:1775764836.261297:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899e00. 00000080:00000010:3.1:1775764836.282807:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2a20. 00000080:00000010:3.1:1775764836.282839:0:14453:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942276298c0. 00010000:00000010:3.1:1775764836.307214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917200. 00000080:00000010:0.1:1775764836.307551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f860. 00000080:00000010:0.1:1775764836.355239:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2620. 00010000:00000010:3.1:1775764836.388228:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cbd800. 00010000:00000010:0.1:1775764836.395421:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898000. 00010000:00000010:3.1:1775764836.426138:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420794c100. 00010000:00000010:3.1:1775764836.426171:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917400. 00010000:00000010:1.1:1775764836.442196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc200. 00010000:00000010:3.1:1775764836.476278:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943500. 00010000:00000010:3.1:1775764836.476286:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ce1800. 00010000:00000010:3.1:1775764836.481295:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916a00. 00000080:00000010:3.1:1775764836.493530:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db23e0. 00010000:00000010:1.1:1775764836.500290:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917400. 00010000:00000010:1.1:1775764836.547288:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917a00. 00010000:00000010:3.1:1775764836.548181:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898400. 00010000:00000010:3.1:1775764836.610365:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8200. 00010000:00000010:0.1:1775764836.648215:0:6158:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917800. 00010000:00000010:3.1:1775764836.648629:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ade00. 00010000:00000010:3.1:1775764836.648635:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0a00. 00010000:00000010:1.1:1775764836.648996:0:6159:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898400. 00010000:00000010:0.1:1775764836.702195:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8800. 00010000:00000010:3.1:1775764836.712718:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916200. 00010000:00000010:3.1:1775764836.731204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917200. 00010000:00000010:3.1:1775764836.739349:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917a00. 00010000:00000010:3.1:1775764836.748198:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420550be00. 00010000:00000010:3.1:1775764836.748206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fcc00. 00010000:00000010:1.1:1775764836.784208:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917a00. 00010000:00000010:3.1:1775764836.813666:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb9c00. 00010000:00000010:3.1:1775764836.813672:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2a00. 00010000:00000010:3.1:1775764836.889219:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc800. 00010000:00000010:0.1:1775764836.918208:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898000. 00010000:00000010:3.1:1775764836.921282:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898800. 00010000:00000010:3.1:1775764836.933245:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc000. 00010000:00000010:0.1:1775764836.954827:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899a00. 00010000:00000010:3.1:1775764836.957369:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fdc00. 00010000:00000010:3.1:1775764836.970797:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0600. 00010000:00000010:3.1:1775764836.990351:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420794c400. 00010000:00000010:3.1:1775764836.990357:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916000. 00010000:00000010:3.1:1775764836.996410:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898800. 00010000:00000010:0.1:1775764837.020439:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793be00. 00010000:00000010:3.1:1775764837.088269:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942079b7300. 00010000:00000010:3.1:1775764837.088291:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994209db2a00. 00010000:00000010:3.1:1775764837.088296:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916000. 00010000:00000010:3.1:1775764837.133205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb9e00. 00010000:00000010:3.1:1775764837.178208:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da800. 00010000:00000010:3.1:1775764837.178218:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793be00. 00010000:00000010:0.1:1775764837.225187:0:14461:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942079b7100. 00010000:00000010:0.1:1775764837.225197:0:14461:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2200. 00010000:00000010:3.1:1775764837.225571:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078adf00. 00010000:00000010:3.1:1775764837.225578:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1a00. 00010000:00000010:3.1:1775764837.247879:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db600. 00010000:00000010:3.1:1775764837.296223:0:14464:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fda00. 00010000:00000010:3.1:1775764837.304321:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d89400. 00010000:00000010:0.1:1775764837.304382:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420550b700. 00010000:00000010:0.1:1775764837.304388:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fde00. 00010000:00000010:3.1:1775764837.404415:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d89000. 00010000:00000010:3.1:1775764837.455191:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916000. 00010000:00000010:3.1:1775764837.461190:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd800. 00010000:00000010:3.1:1775764837.500469:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da600. 00010000:00000010:3.1:1775764837.511289:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943800. 00010000:00000010:3.1:1775764837.511298:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ce1200. 00010000:00000010:0.1:1775764837.538209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dbe00. 00010000:00000010:3.1:1775764837.562326:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916000. 00010000:00000010:3.1:1775764837.593037:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da000. 00000080:00000010:3.1:1775764837.604021:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2d80. 00000080:00000010:3.1:1775764837.604030:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2420. 00010000:00000010:3.1:1775764837.604036:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899000. 00010000:00000010:0.1:1775764837.606550:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fde00. 00010000:00000010:3.1:1775764837.665303:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db400. 00010000:00000010:3.1:1775764837.716228:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078eff00. 00010000:00000010:3.1:1775764837.716241:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dae00. 00010000:00000010:3.1:1775764837.744078:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dbc00. 00010000:00000010:1.1:1775764837.791322:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209d57400. 00010000:00000010:1.1:1775764837.791330:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075cac00. 00010000:00000010:1.1:1775764837.795210:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966100. 00010000:00000010:1.1:1775764837.795225:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6c00. 00000080:00000010:0.1:1775764837.806423:0:14461:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f540. 00010000:00000010:1.1:1775764837.818207:0:2401:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943d00. 00010000:00000010:1.1:1775764837.818214:0:2401:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916400. 00010000:00000010:3.1:1775764837.830099:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994205348a00. 00010000:00000010:3.1:1775764837.830121:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894400. 00010000:00000010:3.1:1775764837.845235:0:2403:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ad400. 00010000:00000010:3.1:1775764837.845246:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0c00. 00010000:00000010:1.1:1775764837.852407:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fda00. 00010000:00000010:1.1:1775764837.895217:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dbe00. 00010000:00000010:3.1:1775764837.910740:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dba00. 00010000:00000010:3.1:1775764837.938009:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db600. 00000080:00000010:0.1:1775764837.949852:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2640. 00010000:00000010:3.1:1775764837.952206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db800. 00000080:00000010:3.1:1775764837.994474:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f1a0. 00010000:00000010:3.1:1775764838.027430:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da600. 00000080:00000010:0.1:1775764838.031243:0:2392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db26e0. 00010000:00000010:3.1:1775764838.080259:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db800. 00010000:00000010:0.1:1775764838.147397:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d89e00. 00010000:00000010:3.1:1775764838.159429:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dba00. 00010000:00000010:3.1:1775764838.232707:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dbe00. 00010000:00000010:3.1:1775764838.254336:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc800. 00010000:00000010:0.1:1775764838.272760:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dbe00. 00010000:00000010:3.1:1775764838.272923:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db600. 00010000:00000010:3.1:1775764838.352828:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917200. 00010000:00000010:0.1:1775764838.375382:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dbc00. 00010000:00000010:3.1:1775764838.430220:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db800. 00010000:00000010:0.1:1775764838.457960:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc800. 00010000:00000010:0.1:1775764838.477413:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917200. 00010000:00000010:3.1:1775764838.506132:0:14473:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970c00. 00010000:00000010:0.1:1775764838.530206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db800. 00010000:00000010:3.1:1775764838.558628:0:6169:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db600. 00010000:00000010:0.1:1775764838.601274:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902400. 00010000:00000010:1.1:1775764838.625733:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971000. 00010000:00000010:0.1:1775764838.710862:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dbe00. 00010000:00000010:0.1:1775764838.710870:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db600. 00010000:00000010:1.1:1775764838.779830:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6000. 00010000:00000010:3.1:1775764838.807384:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fda00. 00010000:00000010:0.1:1775764838.849960:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1200. 00010000:00000010:1.1:1775764838.867226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fda00. 00010000:00000010:0.1:1775764838.870872:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0600. 00010000:00000010:3.1:1775764838.871121:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902800. 00010000:00000010:3.1:1775764838.875237:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db600. 00010000:00000010:1.1:1775764838.942322:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb3200. 00010000:00000010:3.1:1775764838.951120:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ef600. 00010000:00000010:3.1:1775764838.951139:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207535e00. 00010000:00000010:1.1:1775764838.963195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2a00. 00000080:00000010:0.1:1775764838.980395:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a6a0. 00000080:00000010:0.1:1775764838.980405:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a9a0. 00010000:00000010:0.1:1775764838.980434:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d88200. 00010000:00000010:3.1:1775764838.983407:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793d000. 00010000:00000010:3.1:1775764839.042367:0:32:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209d57300. 00010000:00000010:3.1:1775764839.042384:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0800. 00010000:00000010:3.1:1775764839.075952:0:31:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793c600. 00010000:00000010:3.1:1775764839.075961:0:31:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc200. 00000080:00000010:3.1:1775764839.146976:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2480. 00000080:00000010:1.1:1775764839.162211:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2dc0. 00010000:00000010:0.1:1775764839.162278:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc200. 00010000:00000010:3.1:1775764839.162527:0:31:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793d800. 00000080:00000010:1.1:1775764839.180429:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a440. 00010000:00000010:3.1:1775764839.181774:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2800. 00010000:00000010:1.1:1775764839.195259:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0400. 00000080:00000010:1.1:1775764839.258326:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c83a0. 00010000:00000010:3.1:1775764839.269265:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2800. 00010000:00000010:1.1:1775764839.269350:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2a00. 00000080:00000010:1.1:1775764839.269358:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6aa40. 00010000:00000010:1.1:1775764839.322244:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793d800. 00000080:00000010:3.1:1775764839.345204:0:2403:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2140. 00010000:00000010:0.1:1775764839.349349:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793c600. 00010000:00000010:0.1:1775764839.371531:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0400. 00010000:00000010:3.1:1775764839.386396:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc200. 00010000:00000010:1.1:1775764839.410077:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793d000. 00000080:00000010:0.1:1775764839.418850:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f920. 00010000:00000010:0.1:1775764839.435994:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1c00. 00010000:00000010:3.1:1775764839.493556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc200. 00010000:00000010:0.1:1775764839.510204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0c00. 00010000:00000010:3.1:1775764839.537784:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798ce00. 00000080:00000010:3.1:1775764839.541432:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6aa40. 00000080:00000010:3.1:1775764839.541459:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2be0. 00010000:00000010:3.1:1775764839.541468:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420694c400. 00010000:00000010:3.1:1775764839.602443:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420794cc00. 00010000:00000010:3.1:1775764839.602457:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc200. 00000080:00000010:3.1:1775764839.618295:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db29a0. 00010000:00000010:0.1:1775764839.638060:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc200. 00000080:00000010:0.1:1775764839.718952:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c83a0. 00010000:00000010:1.1:1775764839.721526:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0600. 00010000:00000010:1.1:1775764839.728373:0:2400:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2a00. 00000080:00000010:0.1:1775764839.728977:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8920. 00010000:00000010:3.1:1775764839.730232:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0400. 00010000:00000010:3.1:1775764839.798438:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798da00. 00010000:00000010:1.1:1775764839.947182:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994203fb1200. 00010000:00000010:3.1:1775764839.965232:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994203fb0200. 00010000:00000010:3.1:1775764840.058824:0:6169:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994203fb0200. 00010000:00000010:3.1:1775764840.065383:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fca00. 00010000:00000010:3.1:1775764840.141337:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1200. 00010000:00000010:0.1:1775764840.149199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994203fb0200. 00000080:00000010:0.1:1775764840.149207:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2ca0. 00010000:00000010:0.1:1775764840.172211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d800. 00010000:00000010:3.1:1775764840.195348:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798ce00. 00010000:00000010:1.1:1775764840.197185:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079ca400. 00010000:00000010:3.1:1775764840.239633:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fca00. 00010000:00000010:3.1:1775764840.256276:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da000. 00010000:00000010:3.1:1775764840.298336:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2a00. 00000080:00000010:3.1:1775764840.352249:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8ec0. 00000080:00000010:1.1:1775764840.376199:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f540. 00000080:00000010:1.1:1775764840.376217:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db21a0. 00000080:00000010:1.1:1775764840.388282:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8140. 00000080:00000010:1.1:1775764840.397211:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2e80. 00000080:00000010:1.1:1775764840.397236:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6ad60. 00000080:00000010:0.1:1775764840.406209:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6aac0. 00000080:00000010:1.1:1775764840.418501:0:19:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a0e0. 00000080:00000010:3.1:1775764840.429356:0:13:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8320. 00000080:00000010:1.1:1775764840.433865:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8520. 00000080:00000010:3.1:1775764840.448645:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f800. 00000080:00000010:3.1:1775764840.463326:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756fea0. 00000080:00000010:3.1:1775764840.463357:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2720. 00000080:00000010:3.1:1775764840.477640:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2ea0. 00000080:00000010:3.1:1775764840.477676:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2ec0. 00000080:00000010:3.1:1775764840.482297:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8860. 00000080:00000010:3.1:1775764840.482318:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff99422762c200. 00010000:00000010:3.1:1775764840.522592:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1800. 00000080:00000010:0.1:1775764840.523739:0:12:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756f820. 00000080:00000010:0.1:1775764840.523751:0:12:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6aea0. 00010000:00000010:0.1:1775764840.523758:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079cac00. 00010000:00000010:0.1:1775764840.523764:0:12:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078efb00. 00010000:00000010:0.1:1775764840.523770:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db400. 00010000:00000010:0.1:1775764840.523774:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb3a00. 00000080:00000010:3.1:1775764840.534444:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2540. 00010000:00000010:3.1:1775764840.574128:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079cbe00. 00010000:00000010:3.1:1775764840.575449:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0800. 00010000:00000010:3.1:1775764840.579346:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079ca200. 00010000:00000010:3.1:1775764840.595041:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb3800. 00010000:00000010:3.1:1775764840.606345:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079ca800. 00010000:00000010:3.1:1775764840.606357:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420794cb00. 00010000:00000010:3.1:1775764840.606403:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1600. 00000080:00000010:3.1:1775764840.624946:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2a20. 00000080:00000010:3.1:1775764840.624987:0:14454:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d5ac0. 00010000:00000010:0.1:1775764840.644116:0:2400:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb3800. 00010000:00000010:3.1:1775764840.654198:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994203fb0600. 00000080:00000010:3.1:1775764840.698576:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756ff20. 00010000:00000010:3.1:1775764840.717245:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb3a00. 00010000:00000010:3.1:1775764840.744296:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb3200. 00010000:00000010:3.1:1775764840.781290:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb3a00. 00000080:00000010:3.1:1775764840.792476:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6af00. 00010000:00000010:1.1:1775764840.793299:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc800. 00000080:00000010:0.1:1775764840.793604:0:13:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db24e0. 00010000:00000010:3.1:1775764840.831292:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c200. 00010000:00000010:3.1:1775764840.831308:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ef600. 00010000:00000010:3.1:1775764840.831313:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2600. 00000080:00000010:0.1:1775764840.865260:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942189918c0. 00000080:00000010:3.1:1775764840.865283:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8960. 00010000:00000010:3.1:1775764840.989216:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798ce00. 00010000:00000010:3.1:1775764841.008224:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dba00. 00010000:00000010:3.1:1775764841.054380:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fdc00. 00010000:00000010:3.1:1775764841.089772:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079cbe00. 00010000:00000010:0.1:1775764841.168455:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb3a00. 00010000:00000010:3.1:1775764841.189317:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2600. 00010000:00000010:0.1:1775764841.209307:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2a00. 00010000:00000010:3.1:1775764841.244070:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc200. 00010000:00000010:1.1:1775764841.272258:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fda00. 00010000:00000010:1.1:1775764841.297846:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2a00. 00010000:00000010:1.1:1775764841.297856:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb3200. 00010000:00000010:3.1:1775764841.322427:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d89400. 00010000:00000010:3.1:1775764841.358212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2600. 00010000:00000010:1.1:1775764841.362307:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc200. 00010000:00000010:1.1:1775764841.423228:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc200. 00010000:00000010:3.1:1775764841.439480:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dbc00. 00010000:00000010:3.1:1775764841.447905:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917a00. 00010000:00000010:3.1:1775764841.449202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0800. 00010000:00000010:3.1:1775764841.487217:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1800. 00010000:00000010:0.1:1775764841.498202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079ca400. 00010000:00000010:0.1:1775764841.510245:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1800. 00010000:00000010:3.1:1775764841.515393:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fcc00. 00010000:00000010:3.1:1775764841.528051:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db800. 00010000:00000010:3.1:1775764841.538299:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb3200. 00010000:00000010:3.1:1775764841.577263:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994203fb0200. 00010000:00000010:3.1:1775764841.581219:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2a00. 00010000:00000010:3.1:1775764841.585309:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0e00. 00010000:00000010:3.1:1775764841.663202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dbe00. 00010000:00000010:3.1:1775764841.715205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d800. 00010000:00000010:0.1:1775764841.735174:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798ca00. 00010000:00000010:0.1:1775764841.769925:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81c00. 00010000:00000010:3.1:1775764841.815194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0e00. 00010000:00000010:3.1:1775764841.821032:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81800. 00010000:00000010:1.1:1775764841.824252:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80000. 00010000:00000010:0.1:1775764841.861319:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1c00. 00010000:00000010:3.1:1775764841.916225:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80c00. 00010000:00000010:3.1:1775764841.933691:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80a00. 00010000:00000010:3.1:1775764841.933700:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81a00. 00010000:00000010:1.1:1775764841.944036:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81200. 00010000:00000010:1.1:1775764841.944051:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0e00. 00000080:00000010:3.1:1775764841.945088:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2640. 00010000:00000010:3.1:1775764841.954424:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d400. 00010000:00000010:3.1:1775764841.964192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81800. 00010000:00000010:3.1:1775764841.985198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd400. 00010000:00000010:3.1:1775764841.985203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80000. 00010000:00000010:3.1:1775764842.032950:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81200. 00010000:00000010:3.1:1775764842.039535:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81c00. 00010000:00000010:3.1:1775764842.043324:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898400. 00010000:00000010:0.1:1775764842.054199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079cbc00. 00010000:00000010:3.1:1775764842.096751:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80000. 00010000:00000010:1.1:1775764842.127199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80000. 00010000:00000010:3.1:1775764842.135818:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80a00. 00010000:00000010:3.1:1775764842.161298:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789d600. 00010000:00000010:3.1:1775764842.163745:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209d57600. 00010000:00000010:3.1:1775764842.163754:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d89c00. 00010000:00000010:3.1:1775764842.182137:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789dc00. 00010000:00000010:3.1:1775764842.182148:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789c000. 00010000:00000010:0.1:1775764842.185243:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420794c400. 00010000:00000010:0.1:1775764842.185262:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079daa00. 00010000:00000010:3.1:1775764842.207727:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789c200. 00010000:00000010:3.1:1775764842.218464:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420550b200. 00010000:00000010:3.1:1775764842.218478:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd200. 00010000:00000010:3.1:1775764842.234270:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789d400. 00010000:00000010:3.1:1775764842.246983:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789d800. 00010000:00000010:3.1:1775764842.275246:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789ce00. 00010000:00000010:0.1:1775764842.287739:0:2400:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798ce00. 00010000:00000010:0.1:1775764842.287746:0:2400:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078efe00. 00010000:00000010:0.1:1775764842.287750:0:2400:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da200. 00010000:00000010:0.1:1775764842.296456:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209d57800. 00010000:00000010:0.1:1775764842.296463:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d88000. 00010000:00000010:3.1:1775764842.306920:0:14464:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789da00. 00010000:00000010:3.1:1775764842.313243:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789d800. 00010000:00000010:3.1:1775764842.354107:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d400. 00010000:00000010:0.1:1775764842.415928:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0e00. 00010000:00000010:3.1:1775764842.423352:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798da00. 00010000:00000010:0.1:1775764842.430017:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789c000. 00010000:00000010:3.1:1775764842.433782:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ef900. 00010000:00000010:3.1:1775764842.433795:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916a00. 00010000:00000010:1.1:1775764842.464726:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c200. 00010000:00000010:3.1:1775764842.472493:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789c200. 00010000:00000010:3.1:1775764842.472537:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420550bd00. 00010000:00000010:3.1:1775764842.472544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fce00. 00010000:00000010:3.1:1775764842.496194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1600. 00010000:00000010:1.1:1775764842.520365:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0e00. 00010000:00000010:3.1:1775764842.546201:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789d600. 00010000:00000010:0.1:1775764842.550399:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ef200. 00010000:00000010:0.1:1775764842.550414:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00010000:00000010:3.1:1775764842.562785:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209d57900. 00010000:00000010:3.1:1775764842.562795:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8000. 00010000:00000010:3.1:1775764842.589212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798cc00. 00010000:00000010:3.1:1775764842.599204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789c200. 00010000:00000010:1.1:1775764842.619112:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2a00. 00010000:00000010:3.1:1775764842.656195:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420794c100. 00010000:00000010:3.1:1775764842.656211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899800. 00010000:00000010:3.1:1775764842.752524:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789c000. 00010000:00000010:1.1:1775764842.762869:0:14461:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079ca800. 00010000:00000010:3.1:1775764842.771187:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420550b300. 00010000:00000010:3.1:1775764842.771195:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd600. 00010000:00000010:0.1:1775764842.844212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789da00. 00010000:00000010:3.1:1775764842.878594:0:32:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ef500. 00010000:00000010:3.1:1775764842.878610:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899c00. 00010000:00000010:1.1:1775764842.899358:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb3200. 00010000:00000010:3.1:1775764842.942619:0:2391:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789c200. 00010000:00000010:1.1:1775764842.968206:0:1026:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789d600. 00010000:00000010:3.1:1775764842.977261:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb3200. 00010000:00000010:0.1:1775764842.978205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079cbe00. 00010000:00000010:3.1:1775764843.033706:0:1026:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420694dc00. 00010000:00000010:1.1:1775764843.048198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d800. 00010000:00000010:1.1:1775764843.058207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079cbc00. 00010000:00000010:0.1:1775764843.058274:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789d600. 00010000:00000010:3.1:1775764843.109244:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420694dc00. 00000080:00000010:3.1:1775764843.121221:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bc80. 00010000:00000010:0.1:1775764843.138371:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d000. 00010000:00000010:0.1:1775764843.158275:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798cc00. 00010000:00000010:3.1:1775764843.176326:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789ce00. 00010000:00000010:0.1:1775764843.195999:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079cbe00. 00000080:00000010:0.1:1775764843.206284:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8b60. 00000080:00000010:0.1:1775764843.206293:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bde0. 00000080:00000010:0.1:1775764843.206302:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2d60. 00000080:00000010:0.1:1775764843.206308:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2920. 00000080:00000010:0.1:1775764843.206312:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2c20. 00000080:00000010:0.1:1775764843.206315:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2180. 00010000:00000010:0.1:1775764843.206319:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb3800. 00010000:00000010:3.1:1775764843.245471:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789d600. 00010000:00000010:0.1:1775764843.267691:0:2393:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420794c900. 00010000:00000010:0.1:1775764843.267701:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079cb400. 00000080:00000010:3.1:1775764843.296090:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2b60. 00010000:00000010:3.1:1775764843.305670:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798da00. 00010000:00000010:3.1:1775764843.334319:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079cbe00. 00000080:00000010:1.1:1775764843.374663:0:13:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2120. 00010000:00000010:0.1:1775764843.377321:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798ca00. 00010000:00000010:0.1:1775764843.377332:0:2393:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420794c900. 00010000:00000010:0.1:1775764843.377338:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8800. 00010000:00000010:3.1:1775764843.386393:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789ce00. 00010000:00000010:3.1:1775764843.393900:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798da00. 00000080:00000010:0.1:1775764843.472570:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8920. 00000080:00000010:3.1:1775764843.477035:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2f80. 00010000:00000010:3.1:1775764843.488145:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb3200. 00010000:00000010:3.1:1775764843.488167:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420694dc00. 00010000:00000010:3.1:1775764843.510335:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8000. 00010000:00000010:3.1:1775764843.545202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8200. 00010000:00000010:3.1:1775764843.559198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb9c00. 00010000:00000010:3.1:1775764843.571769:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8200. 00010000:00000010:3.1:1775764843.596844:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb3200. 00010000:00000010:3.1:1775764843.616211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d000. 00010000:00000010:3.1:1775764843.641344:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207534800. 00000080:00000010:3.1:1775764843.657892:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c81e0. 00000080:00000010:0.1:1775764843.662893:0:2392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2040. 00010000:00000010:1.1:1775764843.663433:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb3200. 00010000:00000010:0.1:1775764843.676490:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079cac00. 00000080:00000010:0.1:1775764843.676516:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8d80. 00010000:00000010:3.1:1775764843.684622:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207535e00. 00010000:00000010:1.1:1775764843.762888:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0600. 00010000:00000010:0.1:1775764843.793210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8000. 00010000:00000010:3.1:1775764843.838209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb3200. 00000080:00000010:0.1:1775764843.856323:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2b20. 00010000:00000010:1.1:1775764843.860212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079ca400. 00010000:00000010:1.1:1775764843.884567:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207535e00. 00010000:00000010:3.1:1775764843.889970:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8000. 00010000:00000010:3.1:1775764843.908698:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079cbe00. 00000080:00000010:0.1:1775764843.923219:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b080. 00010000:00000010:1.1:1775764843.933670:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079cbc00. 00010000:00000010:1.1:1775764843.955521:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8000. 00010000:00000010:3.1:1775764843.961911:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb9c00. 00010000:00000010:3.1:1775764843.971602:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079ca800. 00010000:00000010:3.1:1775764844.005207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420694c400. 00010000:00000010:3.1:1775764844.022887:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079cbc00. 00000080:00000010:1.1:1775764844.026802:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6abc0. 00010000:00000010:3.1:1775764844.028409:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798cc00. 00010000:00000010:3.1:1775764844.049233:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb9e00. 00000080:00000010:0.1:1775764844.093930:0:14464:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2bc0. 00000080:00000010:3.1:1775764844.110977:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bba0. 00010000:00000010:0.1:1775764844.114253:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207534800. 00000080:00000010:0.1:1775764844.155272:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a460. 00000080:00000010:0.1:1775764844.181575:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2b00. 00000080:00000010:3.1:1775764844.191712:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db26c0. 00000080:00000010:3.1:1775764844.199289:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2220. 00000080:00000010:3.1:1775764844.207775:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2560. 00000080:00000010:3.1:1775764844.232384:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8520. 00000080:00000010:3.1:1775764844.255335:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2160. 00000080:00000010:3.1:1775764844.278419:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db21e0. 00000080:00000010:3.1:1775764844.278432:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2300. 00010000:00000010:3.1:1775764844.278438:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420694dc00. 00010000:00000010:3.1:1775764844.278444:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ef700. 00010000:00000010:3.1:1775764844.278449:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079cb400. 00010000:00000010:3.1:1775764844.278452:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798ce00. 00000080:00000010:3.1:1775764844.278455:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db28e0. 00000080:00000010:0.1:1775764844.285386:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db22c0. 00010000:00000010:3.1:1775764844.286215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cbce00. 00000080:00000010:0.1:1775764844.296301:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2100. 00000080:00000010:0.1:1775764844.301698:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db22a0. 00010000:00000010:3.1:1775764844.305140:0:32:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994211bdb500. 00010000:00000010:3.1:1775764844.305171:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898600. 00000080:00000010:0.1:1775764844.305185:0:2392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2960. 00010000:00000010:0.1:1775764844.346512:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079ca400. 00010000:00000010:3.1:1775764844.363904:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80c00. 00010000:00000010:3.1:1775764844.389179:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079cac00. 00010000:00000010:3.1:1775764844.389188:0:2403:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209d57e00. 00010000:00000010:3.1:1775764844.389210:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a7600. 00010000:00000010:3.1:1775764844.408255:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079cb400. 00010000:00000010:3.1:1775764844.430914:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079cac00. 00010000:00000010:3.1:1775764844.464287:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079ca400. 00010000:00000010:3.1:1775764844.472308:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81800. 00010000:00000010:3.1:1775764844.497330:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207535e00. 00010000:00000010:3.1:1775764844.538259:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079cb400. 00010000:00000010:3.1:1775764844.538281:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cbca00. 00010000:00000010:3.1:1775764844.561495:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0400. 00010000:00000010:3.1:1775764844.586527:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079ca400. 00010000:00000010:3.1:1775764844.586533:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cbcc00. 00010000:00000010:1.1:1775764844.613480:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079cb400. 00010000:00000010:3.1:1775764844.626750:0:14473:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078efc00. 00010000:00000010:3.1:1775764844.626767:0:14473:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81200. 00010000:00000010:0.1:1775764844.635860:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cbd800. 00010000:00000010:0.1:1775764844.667663:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079cac00. 00010000:00000010:3.1:1775764844.676556:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cbca00. 00010000:00000010:3.1:1775764844.683199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cbcc00. 00010000:00000010:1.1:1775764844.700200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cbd800. 00010000:00000010:3.1:1775764844.719214:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cbce00. 00010000:00000010:3.1:1775764844.728367:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078efb00. 00010000:00000010:3.1:1775764844.728375:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789d400. 00010000:00000010:1.1:1775764844.741476:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cbcc00. 00010000:00000010:0.1:1775764844.745179:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dac00. 00010000:00000010:3.1:1775764844.759642:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209d57600. 00010000:00000010:3.1:1775764844.759661:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079cb000. 00010000:00000010:3.1:1775764844.764259:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c200. 00010000:00000010:3.1:1775764844.776217:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cbca00. 00010000:00000010:3.1:1775764844.794216:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cbcc00. 00010000:00000010:3.1:1775764844.806938:0:14473:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb9e00. 00010000:00000010:3.1:1775764844.823261:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420794cb00. 00010000:00000010:3.1:1775764844.823270:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0c00. 00010000:00000010:3.1:1775764844.835229:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8000. 00010000:00000010:3.1:1775764844.839358:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8200. 00010000:00000010:3.1:1775764844.839378:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420794c400. 00010000:00000010:3.1:1775764844.839383:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1800. 00010000:00000010:3.1:1775764844.870377:0:32:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420794c800. 00010000:00000010:3.1:1775764844.870385:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1c00. 00010000:00000010:3.1:1775764844.876921:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cbd800. 00010000:00000010:3.1:1775764844.889720:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420550bf00. 00010000:00000010:3.1:1775764844.889728:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d400. 00010000:00000010:3.1:1775764844.910430:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079cac00. 00010000:00000010:0.1:1775764844.917457:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0400. 00010000:00000010:1.1:1775764844.926435:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cbd800. 00010000:00000010:3.1:1775764844.943549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0600. 00010000:00000010:3.1:1775764844.966359:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ce0200. 00010000:00000010:3.1:1775764844.966380:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0600. 00010000:00000010:3.1:1775764845.006831:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420794c000. 00010000:00000010:3.1:1775764845.006846:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1600. 00010000:00000010:0.1:1775764845.010241:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209d57c00. 00010000:00000010:0.1:1775764845.010260:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8600. 00010000:00000010:0.1:1775764845.010264:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209d57000. 00010000:00000010:0.1:1775764845.010268:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dae00. 00010000:00000010:1.1:1775764845.014224:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ef100. 00010000:00000010:1.1:1775764845.014249:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916000. 00010000:00000010:1.1:1775764845.014254:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420794c700. 00010000:00000010:1.1:1775764845.014258:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2800. 00010000:00000010:3.1:1775764845.014828:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ce0800. 00010000:00000010:3.1:1775764845.014836:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ef000. 00010000:00000010:3.1:1775764845.014840:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0a00. 00010000:00000010:3.1:1775764845.014842:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078eff00. 00010000:00000010:3.1:1775764845.014846:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd800. 00010000:00000010:3.1:1775764845.014851:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209d57400. 00010000:00000010:3.1:1775764845.014855:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2200. 00010000:00000010:3.1:1775764845.014857:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209d57200. 00010000:00000010:3.1:1775764845.014861:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970600. 00010000:00000010:3.1:1775764845.041698:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ce0200. 00010000:00000010:0.1:1775764845.053719:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ce1800. 00010000:00000010:3.1:1775764845.079315:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ce0800. 00010000:00000010:1.1:1775764845.102222:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ce0200. 00010000:00000010:3.1:1775764845.152846:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80a00. 00010000:00000010:3.1:1775764845.175196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971000. 00010000:00000010:3.1:1775764845.239091:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80000. 00010000:00000010:0.1:1775764845.254232:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81800. 00010000:00000010:3.1:1775764845.287237:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81200. 00010000:00000010:3.1:1775764845.337239:0:2403:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ada00. 00010000:00000010:3.1:1775764845.337249:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970600. 00010000:00000010:3.1:1775764845.374394:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971a00. 00010000:00000010:3.1:1775764845.450922:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ef000. 00010000:00000010:3.1:1775764845.450931:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970e00. 00010000:00000010:3.1:1775764845.459555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970000. 00010000:00000010:3.1:1775764845.466467:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971600. 00010000:00000010:3.1:1775764845.479381:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970200. 00010000:00000010:3.1:1775764845.534826:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0200. 00000080:00000010:3.1:1775764845.569348:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2640. 00010000:00000010:3.1:1775764845.582199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81800. 00010000:00000010:3.1:1775764845.633602:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970400. 00010000:00000010:0.1:1775764845.657204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971a00. 00010000:00000010:1.1:1775764845.696308:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2200. 00000080:00000010:1.1:1775764845.696314:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a080. 00010000:00000010:3.1:1775764845.718313:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dbe00. 00010000:00000010:3.1:1775764845.718324:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db800. 00010000:00000010:3.1:1775764845.729268:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970200. 00010000:00000010:3.1:1775764845.769199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0200. 00010000:00000010:3.1:1775764845.769211:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ef100. 00010000:00000010:3.1:1775764845.769216:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970000. 00010000:00000010:3.1:1775764845.812467:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970000. 00010000:00000010:1.1:1775764845.813356:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1600. 00000080:00000010:1.1:1775764845.864133:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bdc0. 00010000:00000010:3.1:1775764845.907983:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970600. 00010000:00000010:3.1:1775764845.914246:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2800. 00010000:00000010:3.1:1775764845.914253:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2a00. 00010000:00000010:3.1:1775764845.918233:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1800. 00000080:00000010:3.1:1775764845.942190:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a460. 00010000:00000010:0.1:1775764845.959251:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fce00. 00010000:00000010:3.1:1775764845.967665:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1a00. 00010000:00000010:3.1:1775764845.974213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970400. 00010000:00000010:0.1:1775764846.000597:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970600. 00000080:00000010:0.1:1775764846.025219:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2180. 00000080:00000010:1.1:1775764846.043314:0:2392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2420. 00010000:00000010:1.1:1775764846.052230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971600. 00000080:00000010:3.1:1775764846.052435:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2240. 00000080:00000010:3.1:1775764846.075463:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2c20. 00000080:00000010:3.1:1775764846.100289:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a660. 00000080:00000010:3.1:1775764846.106205:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b0c0. 00000080:00000010:3.1:1775764846.106234:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8020. 00000080:00000010:3.1:1775764846.119341:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b740. 00000080:00000010:0.1:1775764846.130243:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2e00. 00000080:00000010:0.1:1775764846.138189:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bc00. 00010000:00000010:0.1:1775764846.138208:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0200. 00010000:00000010:0.1:1775764846.138214:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ef500. 00010000:00000010:0.1:1775764846.138218:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80a00. 00000080:00000010:0.1:1775764846.151289:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2e40. 00000080:00000010:1.1:1775764846.156224:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994218991080. 00000080:00000010:0.1:1775764846.158603:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8160. 00010000:00000010:3.1:1775764846.203983:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2200. 00010000:00000010:3.1:1775764846.225927:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1000. 00010000:00000010:1.1:1775764846.281795:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1600. 00010000:00000010:1.1:1775764846.281814:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1a00. 00010000:00000010:3.1:1775764846.288028:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db800. 00010000:00000010:3.1:1775764846.323399:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970000. 00000080:00000010:3.1:1775764846.323405:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a5c0. 00010000:00000010:3.1:1775764846.347246:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970e00. 00010000:00000010:3.1:1775764846.353729:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1c00. 00000080:00000010:0.1:1775764846.377791:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d6b40. 00000080:00000010:1.1:1775764846.378892:0:14461:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a340. 00010000:00000010:3.1:1775764846.391617:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d200. 00010000:00000010:3.1:1775764846.391624:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970000. 00010000:00000010:3.1:1775764846.404113:0:6169:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798dc00. 00010000:00000010:3.1:1775764846.429199:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970200. 00010000:00000010:3.1:1775764846.472273:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970600. 00010000:00000010:1.1:1775764846.485191:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7200. 00010000:00000010:1.1:1775764846.493250:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970400. 00010000:00000010:1.1:1775764846.493274:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1a00. 00010000:00000010:3.1:1775764846.496497:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6400. 00010000:00000010:3.1:1775764846.545213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970600. 00010000:00000010:1.1:1775764846.594208:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dbc00. 00010000:00000010:3.1:1775764846.626195:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7a00. 00010000:00000010:3.1:1775764846.648723:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970200. 00010000:00000010:3.1:1775764846.670283:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970000. 00000080:00000010:3.1:1775764846.670294:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2320. 00010000:00000010:3.1:1775764846.684872:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da600. 00000080:00000010:3.1:1775764846.684881:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b560. 00010000:00000010:3.1:1775764846.684902:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dba00. 00000080:00000010:3.1:1775764846.684905:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6ba00. 00010000:00000010:1.1:1775764846.728365:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dba00. 00010000:00000010:1.1:1775764846.741484:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd800. 00010000:00000010:3.1:1775764846.789352:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da600. 00010000:00000010:3.1:1775764846.810979:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970000. 00010000:00000010:3.1:1775764846.830394:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db200. 00010000:00000010:0.1:1775764846.837244:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1a00. 00010000:00000010:0.1:1775764846.837259:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0a00. 00010000:00000010:3.1:1775764846.891370:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970600. 00010000:00000010:3.1:1775764846.908385:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd200. 00010000:00000010:1.1:1775764846.909661:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc200. 00000080:00000010:3.1:1775764846.974567:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2540. 00000080:00000010:3.1:1775764846.992483:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2ec0. 00000080:00000010:3.1:1775764846.992514:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8ec0. 00010000:00000010:3.1:1775764846.992524:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ada00. 00010000:00000010:3.1:1775764846.992530:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970e00. 00000080:00000010:3.1:1775764847.001515:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a840. 00000080:00000010:3.1:1775764847.001538:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994218993180. 00010000:00000010:3.1:1775764847.018795:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db200. 00000080:00000010:3.1:1775764847.027196:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b020. 00010000:00000010:0.1:1775764847.027253:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970200. 00010000:00000010:3.1:1775764847.054189:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970000. 00000080:00000010:3.1:1775764847.077188:0:2402:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b0c0. 00010000:00000010:3.1:1775764847.077208:0:2402:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ef700. 00010000:00000010:3.1:1775764847.077214:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1a00. 00000080:00000010:3.1:1775764847.094237:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2520. 00010000:00000010:3.1:1775764847.117213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970200. 00000080:00000010:3.1:1775764847.117221:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2120. 00010000:00000010:1.1:1775764847.165927:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ada00. 00010000:00000010:1.1:1775764847.165944:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970600. 00010000:00000010:3.1:1775764847.210594:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da800. 00000080:00000010:3.1:1775764847.307587:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2820. 00000080:00000010:0.1:1775764847.367934:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2b80. 00010000:00000010:3.1:1775764847.381421:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420b14ac00. 00010000:00000010:3.1:1775764847.440312:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420b14a000. 00010000:00000010:3.1:1775764847.452718:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994209db3000. 00000080:00000010:1.1:1775764847.478217:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8160. 00010000:00000010:3.1:1775764847.482469:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc000. 00000080:00000010:3.1:1775764847.540198:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2840. 00010000:00000010:3.1:1775764847.556903:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075ca800. 00010000:00000010:1.1:1775764847.580206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894600. 00010000:00000010:0.1:1775764847.614203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895200. 00000080:00000010:1.1:1775764847.654414:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2f80. 00010000:00000010:3.1:1775764847.654703:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db600. 00010000:00000010:0.1:1775764847.678203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fcc00. 00010000:00000010:1.1:1775764847.708217:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd600. 00010000:00000010:1.1:1775764847.731402:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd400. 00010000:00000010:1.1:1775764847.731408:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894400. 00010000:00000010:3.1:1775764847.746553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895800. 00010000:00000010:3.1:1775764847.858868:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420b14ac00. 00010000:00000010:3.1:1775764847.879207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db600. 00010000:00000010:0.1:1775764847.879314:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894a00. 00000080:00000010:0.1:1775764847.879322:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2ea0. 00010000:00000010:0.1:1775764847.946818:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420b14a000. 00010000:00000010:0.1:1775764847.946876:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420b14a400. 00010000:00000010:0.1:1775764848.001766:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420b14b600. 00000080:00000010:0.1:1775764848.027732:0:2392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2720. 00010000:00000010:0.1:1775764848.027748:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca0c00. 00010000:00000010:3.1:1775764848.038652:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895400. 00010000:00000010:3.1:1775764848.095703:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca0e00. 00000080:00000010:0.1:1775764848.096133:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8dc0. 00010000:00000010:0.1:1775764848.096147:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca0c00. 00000080:00000010:3.1:1775764848.118788:0:2402:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2de0. 00000080:00000010:0.1:1775764848.126367:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db25e0. 00010000:00000010:0.1:1775764848.139198:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894200. 00010000:00000010:0.1:1775764848.139209:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420794c700. 00010000:00000010:0.1:1775764848.139214:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970200. 00010000:00000010:3.1:1775764848.141245:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a6600. 00000080:00000010:0.1:1775764848.169898:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8dc0. 00010000:00000010:3.1:1775764848.176292:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895e00. 00010000:00000010:3.1:1775764848.183500:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd600. 00000080:00000010:1.1:1775764848.230271:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a420. 00010000:00000010:3.1:1775764848.238393:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fce00. 00010000:00000010:1.1:1775764848.238771:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fcc00. 00000080:00000010:3.1:1775764848.252138:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2e60. 00000080:00000010:1.1:1775764848.265202:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c82c0. 00010000:00000010:1.1:1775764848.268394:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db600. 00010000:00000010:1.1:1775764848.302623:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894a00. 00010000:00000010:0.1:1775764848.313065:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc000. 00010000:00000010:0.1:1775764848.313079:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420794ce00. 00010000:00000010:0.1:1775764848.313083:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994209db2a00. 00010000:00000010:3.1:1775764848.336379:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da800. 00010000:00000010:3.1:1775764848.340228:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895400. 00010000:00000010:3.1:1775764848.340243:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894200. 00010000:00000010:1.1:1775764848.367724:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895200. 00010000:00000010:3.1:1775764848.379184:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a7600. 00000080:00000010:0.1:1775764848.396544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2ee0. 00010000:00000010:3.1:1775764848.422252:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da600. 00010000:00000010:3.1:1775764848.422314:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da800. 00010000:00000010:3.1:1775764848.493198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da800. 00010000:00000010:3.1:1775764848.493233:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895800. 00010000:00000010:3.1:1775764848.493237:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894600. 00000080:00000010:3.1:1775764848.501692:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b200. 00000080:00000010:3.1:1775764848.501710:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c82e0. 00010000:00000010:0.1:1775764848.506307:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db400. 00010000:00000010:3.1:1775764848.516947:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894200. 00000080:00000010:3.1:1775764848.516967:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db23e0. 00010000:00000010:3.1:1775764848.527283:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895400. 00000080:00000010:3.1:1775764848.527306:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2ca0. 00010000:00000010:3.1:1775764848.578697:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dbc00. 00010000:00000010:3.1:1775764848.578734:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dba00. 00010000:00000010:3.1:1775764848.609486:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895400. 00000080:00000010:0.1:1775764848.635274:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6baa0. 00000080:00000010:0.1:1775764848.635301:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994218995280. 00010000:00000010:1.1:1775764848.644193:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a7600. 00000080:00000010:1.1:1775764848.644202:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8b80. 00010000:00000010:0.1:1775764848.676168:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a6600. 00010000:00000010:0.1:1775764848.676573:0:2400:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc000. 00010000:00000010:3.1:1775764848.701094:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fce00. 00010000:00000010:0.1:1775764848.745262:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420794c700. 00010000:00000010:0.1:1775764848.745279:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895800. 00010000:00000010:0.1:1775764848.855786:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075dbe00. 00010000:00000010:0.1:1775764848.885201:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ada00. 00010000:00000010:0.1:1775764848.885211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1000. 00010000:00000010:0.1:1775764848.885215:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ef000. 00010000:00000010:0.1:1775764848.885237:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1a00. 00000080:00000010:0.1:1775764848.914632:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8140. 00010000:00000010:0.1:1775764848.914658:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894600. 00010000:00000010:0.1:1775764848.976404:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075cac00. 00010000:00000010:0.1:1775764849.070797:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fcc00. 00000080:00000010:3.1:1775764849.083643:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a500. 00010000:00000010:3.1:1775764849.083667:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc000. 00010000:00000010:3.1:1775764849.097184:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075ca800. 00010000:00000010:3.1:1775764849.123374:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895600. 00010000:00000010:0.1:1775764849.141316:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075cbc00. 00000080:00000010:0.1:1775764849.162059:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2680. 00010000:00000010:3.1:1775764849.188124:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a6a00. 00010000:00000010:1.1:1775764849.188207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894400. 00000080:00000010:3.1:1775764849.191362:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8d00. 00010000:00000010:3.1:1775764849.278853:0:14452:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894600. 00010000:00000010:3.1:1775764849.278859:0:14452:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894400. 00000080:00000010:0.1:1775764849.333393:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2be0. 00010000:00000010:1.1:1775764849.339640:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a7600. 00010000:00000010:0.1:1775764849.393237:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a7600. 00010000:00000010:0.1:1775764849.400414:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd800. 00010000:00000010:0.1:1775764849.400428:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894400. 00000080:00000010:0.1:1775764849.404215:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c81e0. 00000080:00000010:0.1:1775764849.404242:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c83a0. 00010000:00000010:3.1:1775764849.463865:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894600. 00010000:00000010:1.1:1775764849.464202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a7800. 00010000:00000010:1.1:1775764849.473665:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895600. 00010000:00000010:3.1:1775764849.492462:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894400. 00000080:00000010:3.1:1775764849.498955:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2f00. 00010000:00000010:3.1:1775764849.548171:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da200. 00010000:00000010:1.1:1775764849.548666:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dbc00. 00010000:00000010:3.1:1775764849.582373:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dae00. 00000080:00000010:3.1:1775764849.582393:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b2a0. 00010000:00000010:0.1:1775764849.586558:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078b3400. 00010000:00000010:0.1:1775764849.586565:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fcc00. 00010000:00000010:0.1:1775764849.623194:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209d57900. 00010000:00000010:0.1:1775764849.623204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789c000. 00000080:00000010:0.1:1775764849.634401:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6abe0. 00000080:00000010:0.1:1775764849.634427:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942189918c0. 00010000:00000010:0.1:1775764849.650333:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a6400. 00000080:00000010:0.1:1775764849.650355:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8fe0. 00010000:00000010:3.1:1775764849.661314:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789dc00. 00010000:00000010:3.1:1775764849.766368:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899c00. 00000080:00000010:3.1:1775764849.766391:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8fe0. 00010000:00000010:3.1:1775764849.866659:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dbc00. 00010000:00000010:3.1:1775764849.866731:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dac00. 00010000:00000010:1.1:1775764849.875370:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942420d9800. 00010000:00000010:1.1:1775764849.875377:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da600. 00000080:00000010:1.1:1775764849.956532:0:2392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b780. 00000080:00000010:1.1:1775764849.961275:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227600840. 00010000:00000010:3.1:1775764849.961617:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fce00. 00000080:00000010:3.1:1775764849.961624:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a5e0. 00010000:00000010:3.1:1775764849.981284:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dbc00. 00010000:00000010:3.1:1775764849.989245:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dac00. 00010000:00000010:0.1:1775764849.997879:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789c800. 00000080:00000010:0.1:1775764849.997889:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2480. 00010000:00000010:0.1:1775764850.071340:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898c00. 00010000:00000010:0.1:1775764850.072885:0:2401:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899000. 00010000:00000010:3.1:1775764850.123217:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898800. 00010000:00000010:1.1:1775764850.142086:0:2392:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ef000. 00010000:00000010:1.1:1775764850.142100:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898000. 00010000:00000010:3.1:1775764850.177115:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dac00. 00000080:00000010:3.1:1775764850.177124:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b380. 00010000:00000010:3.1:1775764850.255466:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789cc00. 00010000:00000010:3.1:1775764850.266040:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dbc00. 00010000:00000010:0.1:1775764850.325131:0:13:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207976d00. 00010000:00000010:0.1:1775764850.325140:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81200. 00010000:00000010:0.1:1775764850.325144:0:13:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942079b7100. 00010000:00000010:0.1:1775764850.325146:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb3800. 00010000:00000010:1.1:1775764850.345206:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ad400. 00010000:00000010:1.1:1775764850.345217:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971800. 00010000:00000010:1.1:1775764850.345223:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078efa00. 00010000:00000010:1.1:1775764850.345226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971a00. 00010000:00000010:0.1:1775764850.378189:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971a00. 00010000:00000010:1.1:1775764850.442404:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da800. 00010000:00000010:1.1:1775764850.520393:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db200. 00000080:00000010:3.1:1775764850.579318:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b700. 00010000:00000010:3.1:1775764850.579336:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da000. 00010000:00000010:3.1:1775764850.603848:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789cc00. 00000080:00000010:3.1:1775764850.621367:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2be0. 00010000:00000010:1.1:1775764850.635398:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a7600. 00010000:00000010:1.1:1775764850.635411:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420b307e00. 00010000:00000010:1.1:1775764850.635414:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895800. 00010000:00000010:1.1:1775764850.644032:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789c800. 00000080:00000010:1.1:1775764850.644065:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8e80. 00000080:00000010:1.1:1775764850.644087:0:14453:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d3180. 00010000:00000010:3.1:1775764850.703423:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971800. 00000080:00000010:0.1:1775764850.721305:0:2392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a340. 00010000:00000010:3.1:1775764850.735943:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898000. 00010000:00000010:1.1:1775764850.808247:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789c800. 00010000:00000010:1.1:1775764850.820206:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970400. 00010000:00000010:1.1:1775764850.820218:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789c200. 00010000:00000010:3.1:1775764850.825069:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970600. 00010000:00000010:3.1:1775764850.839187:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da000. 00000080:00000010:1.1:1775764850.839299:0:2392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b380. 00010000:00000010:3.1:1775764850.880331:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898000. 00010000:00000010:3.1:1775764850.900383:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da000. 00010000:00000010:0.1:1775764850.922062:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899600. 00000080:00000010:0.1:1775764850.922078:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8fe0. 00010000:00000010:3.1:1775764850.946361:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789cc00. 00010000:00000010:3.1:1775764850.946380:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da000. 00010000:00000010:3.1:1775764850.991244:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da000. 00010000:00000010:3.1:1775764851.003634:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789c200. 00010000:00000010:1.1:1775764851.043203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789c800. 00010000:00000010:1.1:1775764851.043220:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970e00. 00010000:00000010:3.1:1775764851.062054:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899000. 00010000:00000010:3.1:1775764851.062068:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da000. 00010000:00000010:3.1:1775764851.092181:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789c200. 00000080:00000010:0.1:1775764851.098976:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8d80. 00000080:00000010:0.1:1775764851.098986:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b3e0. 00000080:00000010:0.1:1775764851.098996:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2ee0. 00000080:00000010:0.1:1775764851.099004:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db23e0. 00000080:00000010:0.1:1775764851.099008:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2680. 00000080:00000010:0.1:1775764851.099011:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b6e0. 00010000:00000010:0.1:1775764851.099016:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970400. 00010000:00000010:3.1:1775764851.099413:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789cc00. 00010000:00000010:0.1:1775764851.117224:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789c800. 00010000:00000010:1.1:1775764851.147774:0:2392:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078b3b00. 00010000:00000010:1.1:1775764851.147794:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899600. 00010000:00000010:0.1:1775764851.159417:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899a00. 00010000:00000010:0.1:1775764851.184224:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899800. 00000080:00000010:0.1:1775764851.184276:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8b80. 00000080:00000010:0.1:1775764851.184284:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b8e0. 00000080:00000010:0.1:1775764851.184291:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8360. 00000080:00000010:0.1:1775764851.184294:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a500. 00000080:00000010:0.1:1775764851.184300:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db20a0. 00000080:00000010:0.1:1775764851.184306:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db21a0. 00010000:00000010:0.1:1775764851.184310:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898800. 00010000:00000010:3.1:1775764851.212249:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da000. 00010000:00000010:3.1:1775764851.212313:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789cc00. 00010000:00000010:3.1:1775764851.253609:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db200. 00010000:00000010:0.1:1775764851.323203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898400. 00010000:00000010:0.1:1775764851.323251:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075dbe00. 00010000:00000010:3.1:1775764851.328025:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db200. 00010000:00000010:3.1:1775764851.399434:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970400. 00000080:00000010:0.1:1775764851.411653:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6aa40. 00010000:00000010:0.1:1775764851.411670:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075dbe00. 00010000:00000010:3.1:1775764851.414389:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cbd600. 00000080:00000010:0.1:1775764851.461452:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2e60. 00010000:00000010:3.1:1775764851.480471:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cbd800. 00010000:00000010:3.1:1775764851.509427:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971800. 00000080:00000010:3.1:1775764851.509436:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a280. 00010000:00000010:3.1:1775764851.566923:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209d57b00. 00010000:00000010:3.1:1775764851.566943:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db400. 00010000:00000010:3.1:1775764851.566949:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cbd600. 00000080:00000010:3.1:1775764851.566956:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db25e0. 00010000:00000010:3.1:1775764851.608147:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420794c800. 00010000:00000010:3.1:1775764851.608225:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899c00. 00010000:00000010:3.1:1775764851.626774:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db400. 00010000:00000010:3.1:1775764851.640594:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942420d9500. 00010000:00000010:3.1:1775764851.640626:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970200. 00010000:00000010:3.1:1775764851.640631:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db200. 00000080:00000010:3.1:1775764851.651067:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b9a0. 00000080:00000010:3.1:1775764851.651088:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2b60. 00000080:00000010:3.1:1775764851.686350:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8ec0. 00010000:00000010:0.1:1775764851.695630:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dbc00. 00000080:00000010:3.1:1775764851.763453:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b1c0. 00000080:00000010:3.1:1775764851.763474:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b9a0. 00010000:00000010:3.1:1775764851.783908:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207976d00. 00010000:00000010:3.1:1775764851.783921:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dac00. 00000080:00000010:3.1:1775764851.818759:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2840. 00010000:00000010:3.1:1775764851.832239:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970000. 00000080:00000010:3.1:1775764851.832252:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6ad60. 00010000:00000010:3.1:1775764851.846302:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cbd600. 00010000:00000010:3.1:1775764851.851286:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dae00. 00010000:00000010:3.1:1775764851.901357:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dae00. 00010000:00000010:1.1:1775764851.931355:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209d57e00. 00010000:00000010:1.1:1775764851.931364:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dba00. 00010000:00000010:0.1:1775764851.980215:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207931000. 00010000:00000010:0.1:1775764851.980225:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0a00. 00010000:00000010:3.1:1775764852.057566:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0400. 00000080:00000010:3.1:1775764852.074188:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2b80. 00010000:00000010:1.1:1775764852.134134:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1800. 00000080:00000010:3.1:1775764852.134660:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2820. 00000080:00000010:3.1:1775764852.143104:0:14454:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff99422762d280. 00000080:00000010:3.1:1775764852.161118:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2c40. 00010000:00000010:3.1:1775764852.170517:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0e00. 00010000:00000010:0.1:1775764852.183215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da600. 00010000:00000010:3.1:1775764852.208037:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0600. 00000080:00000010:1.1:1775764852.234976:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b1c0. 00010000:00000010:1.1:1775764852.242500:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793c600. 00000080:00000010:1.1:1775764852.242517:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a0e0. 00010000:00000010:3.1:1775764852.245092:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793d000. 00000080:00000010:0.1:1775764852.265209:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2d60. 00000080:00000010:0.1:1775764852.265231:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2660. 00010000:00000010:1.1:1775764852.305212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898c00. 00010000:00000010:0.1:1775764852.305422:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0e00. 00010000:00000010:0.1:1775764852.305429:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793d800. 00010000:00000010:3.1:1775764852.315962:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da600. 00010000:00000010:3.1:1775764852.316024:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1600. 00000080:00000010:3.1:1775764852.316031:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2520. 00010000:00000010:0.1:1775764852.352206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899000. 00010000:00000010:1.1:1775764852.375221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793c600. 00010000:00000010:1.1:1775764852.375285:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dba00. 00010000:00000010:3.1:1775764852.420798:0:32:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942420d9c00. 00010000:00000010:3.1:1775764852.420808:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793d000. 00010000:00000010:3.1:1775764852.434066:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1800. 00000080:00000010:0.1:1775764852.450196:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6af00. 00000080:00000010:0.1:1775764852.450223:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227600840. 00010000:00000010:0.1:1775764852.454278:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899a00. 00000080:00000010:0.1:1775764852.454312:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8ec0. 00010000:00000010:3.1:1775764852.489628:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da600. 00010000:00000010:3.1:1775764852.509971:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898600. 00010000:00000010:0.1:1775764852.535418:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899000. 00010000:00000010:1.1:1775764852.553957:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0400. 00010000:00000010:3.1:1775764852.594247:0:31:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da000. 00010000:00000010:0.1:1775764852.692277:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898000. 00010000:00000010:3.1:1775764852.699493:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1600. 00010000:00000010:3.1:1775764852.729375:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898400. 00010000:00000010:1.1:1775764852.749186:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0e00. 00010000:00000010:3.1:1775764852.766126:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079cac00. 00010000:00000010:3.1:1775764852.786893:0:14473:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899a00. 00010000:00000010:0.1:1775764852.800840:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db800. 00010000:00000010:3.1:1775764852.802098:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971600. 00010000:00000010:3.1:1775764852.818623:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0600. 00010000:00000010:3.1:1775764852.832328:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0e00. 00010000:00000010:1.1:1775764852.856378:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0600. 00010000:00000010:1.1:1775764852.856387:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899c00. 00010000:00000010:1.1:1775764852.888051:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da000. 00010000:00000010:3.1:1775764852.941175:0:1026:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899200. 00010000:00000010:3.1:1775764852.956722:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079ca800. 00000080:00000010:1.1:1775764852.975498:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8d80. 00010000:00000010:0.1:1775764852.975612:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898800. 00010000:00000010:3.1:1775764853.078318:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079ca400. 00000080:00000010:1.1:1775764853.189696:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2700. 00000080:00000010:3.1:1775764853.221299:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a260. 00000080:00000010:3.1:1775764853.231186:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d3180. 00000080:00000010:3.1:1775764853.235037:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2240. 00000080:00000010:3.1:1775764853.235067:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff99422762ca40. 00010000:00000010:3.1:1775764853.235083:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899000. 00000080:00000010:3.1:1775764853.235087:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8920. 00010000:00000010:1.1:1775764853.238437:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079ca000. 00000080:00000010:1.1:1775764853.238461:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a900. 00010000:00000010:3.1:1775764853.283198:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dbc00. 00010000:00000010:3.1:1775764853.305202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8200. 00010000:00000010:3.1:1775764853.350115:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cbd600. 00010000:00000010:3.1:1775764853.403663:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420b307e00. 00010000:00000010:3.1:1775764853.403671:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789c800. 00010000:00000010:0.1:1775764853.428836:0:1358:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420b307300. 00010000:00000010:0.1:1775764853.428845:0:1358:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895400. 00010000:00000010:0.1:1775764853.428849:0:1358:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078a1d00. 00010000:00000010:0.1:1775764853.428853:0:1358:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8000. 00010000:00000010:0.1:1775764853.428858:0:1358:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420794c400. 00010000:00000010:0.1:1775764853.428862:0:1358:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789cc00. 00010000:00000010:0.1:1775764853.428866:0:1358:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942420d9b00. 00010000:00000010:0.1:1775764853.428871:0:1358:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fca00. 00010000:00000010:0.1:1775764853.428873:0:1358:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942420d9900. 00010000:00000010:0.1:1775764853.428877:0:1358:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793d800. 00010000:00000010:0.1:1775764853.429172:0:2401:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078b3400. 00010000:00000010:0.1:1775764853.429176:0:2401:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789c200. 00010000:00000010:0.1:1775764853.429180:0:2401:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ef100. 00010000:00000010:0.1:1775764853.429183:0:2401:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894200. 00010000:00000010:1.1:1775764853.429846:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078a1800. 00010000:00000010:1.1:1775764853.429853:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1600. 00010000:00000010:1.1:1775764853.429858:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209d57100. 00010000:00000010:1.1:1775764853.429862:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971a00. 00010000:00000010:2.1:1775764853.432184:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942069a5f00. 00010000:00000010:2.1:1775764853.432205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994203fb1200. 00010000:00000010:2.1:1775764853.432211:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207976400. 00010000:00000010:2.1:1775764853.432215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898400. 00010000:00000010:3.1:1775764853.460337:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cbd600. 00000080:00000010:3.1:1775764853.460350:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db24e0. 00010000:00000010:2.1:1775764853.470373:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994203fb1200. 00000080:00000010:2.1:1775764853.470388:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2540. 00010000:00000010:2.1:1775764853.506212:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420794c800. 00010000:00000010:2.1:1775764853.506222:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8600. 00010000:00000010:2.1:1775764853.512948:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207976c00. 00010000:00000010:2.1:1775764853.512958:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb9c00. 00010000:00000010:2.1:1775764853.545445:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898800. 00010000:00000010:2.1:1775764853.558209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0e00. 00000080:00000010:0.1:1775764853.558322:0:2392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bda0. 00000080:00000010:3.1:1775764853.571701:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2120. 00010000:00000010:2.1:1775764853.577356:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898000. 00010000:00000010:0.1:1775764853.580444:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db600. 00010000:00000010:2.1:1775764853.581213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079ca400. 00010000:00000010:2.1:1775764853.585262:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078e9000. 00010000:00000010:2.1:1775764853.585280:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898600. 00010000:00000010:2.1:1775764853.585284:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dbc00. 00010000:00000010:3.1:1775764853.602294:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db200. 00010000:00000010:0.1:1775764853.611268:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078a1600. 00010000:00000010:0.1:1775764853.611282:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994203fb1200. 00010000:00000010:0.1:1775764853.611288:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1600. 00000080:00000010:2.1:1775764853.611341:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a260. 00000080:00000010:2.1:1775764853.611352:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8b60. 00000080:00000010:2.1:1775764853.619220:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b3a0. 00000080:00000010:2.1:1775764853.619245:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b7a0. 00010000:00000010:2.1:1775764853.631999:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078a1f00. 00010000:00000010:2.1:1775764853.632008:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0e00. 00010000:00000010:3.1:1775764853.632915:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942420d9f00. 00010000:00000010:3.1:1775764853.632920:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798dc00. 00010000:00000010:2.1:1775764853.650423:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d200. 00010000:00000010:2.1:1775764853.657225:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d400. 00000080:00000010:2.1:1775764853.657233:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2720. 00010000:00000010:2.1:1775764853.661380:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079ca000. 00010000:00000010:2.1:1775764853.686557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079ca800. 00000080:00000010:2.1:1775764853.686567:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a9e0. 00010000:00000010:1.1:1775764853.734403:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d200. 00010000:00000010:2.1:1775764853.743828:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898400. 00010000:00000010:3.1:1775764853.769573:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db200. 00010000:00000010:3.1:1775764853.769583:0:13:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420794c800. 00010000:00000010:3.1:1775764853.769587:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db400. 00010000:00000010:0.1:1775764853.780069:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898600. 00010000:00000010:0.1:1775764853.780078:0:2392:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078e9e00. 00010000:00000010:0.1:1775764853.780084:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798da00. 00010000:00000010:3.1:1775764853.811219:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798dc00. 00010000:00000010:3.1:1775764853.811228:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078a1400. 00010000:00000010:3.1:1775764853.811233:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902400. 00010000:00000010:2.1:1775764853.818204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d200. 00010000:00000010:2.1:1775764853.818214:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078a1f00. 00010000:00000010:2.1:1775764853.818218:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902800. 00010000:00000010:2.1:1775764853.863297:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902800. 00010000:00000010:2.1:1775764853.863357:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d400. 00010000:00000010:1.1:1775764853.868192:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c200. 00010000:00000010:3.1:1775764853.868201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db400. 00010000:00000010:1.1:1775764853.921307:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942420d9700. 00010000:00000010:1.1:1775764853.921323:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079caa00. 00000080:00000010:2.1:1775764853.934750:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b0e0. 00000080:00000010:2.1:1775764853.934776:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227604a40. 00010000:00000010:2.1:1775764853.938180:0:9316:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902800. 00000080:00000010:2.1:1775764853.938194:0:9316:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db26a0. 00010000:00000010:1.1:1775764853.940128:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078a1f00. 00010000:00000010:1.1:1775764853.940137:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902c00. 00000080:00000010:1.1:1775764853.944212:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2720. 00000080:00000010:1.1:1775764853.944247:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d6b40. 00010000:00000010:1.1:1775764853.963137:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079caa00. 00000080:00000010:1.1:1775764853.963168:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6afc0. 00010000:00000010:2.1:1775764854.011360:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994218f6fe00. 00010000:00000010:2.1:1775764854.011422:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903800. 00010000:00000010:2.1:1775764854.011428:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902800. 00010000:00000010:2.1:1775764854.024237:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079ca000. 00010000:00000010:3.1:1775764854.050205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903c00. 00010000:00000010:3.1:1775764854.128208:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902800. 00010000:00000010:3.1:1775764854.214221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db400. 00000080:00000010:3.1:1775764854.214236:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bac0. 00010000:00000010:2.1:1775764854.262203:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899c00. 00010000:00000010:3.1:1775764854.266704:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db400. 00000080:00000010:3.1:1775764854.266714:0:2402:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bac0. 00010000:00000010:2.1:1775764854.272238:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db200. 00010000:00000010:3.1:1775764854.288207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902c00. 00010000:00000010:3.1:1775764854.298747:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da000. 00010000:00000010:2.1:1775764854.308245:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078a1f00. 00010000:00000010:2.1:1775764854.308264:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902800. 00000080:00000010:3.1:1775764854.332319:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c83c0. 00000080:00000010:3.1:1775764854.332358:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d0000. 00010000:00000010:2.1:1775764854.335357:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078e9e00. 00010000:00000010:2.1:1775764854.335375:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db400. 00000080:00000010:2.1:1775764854.344269:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2640. 00000080:00000010:2.1:1775764854.344310:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff99422762c200. 00010000:00000010:3.1:1775764854.352066:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da000. 00000080:00000010:3.1:1775764854.352084:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bac0. 00010000:00000010:2.1:1775764854.370301:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902800. 00000080:00000010:2.1:1775764854.370322:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2960. 00010000:00000010:2.1:1775764854.421207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902800. 00010000:00000010:3.1:1775764854.431654:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902400. 00010000:00000010:3.1:1775764854.431689:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da000. 00010000:00000010:2.1:1775764854.431873:0:25:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db800. 00010000:00000010:3.1:1775764854.466428:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899a00. 00010000:00000010:3.1:1775764854.466441:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994218f6fe00. 00010000:00000010:2.1:1775764854.694186:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899a00. 00000080:00000010:2.1:1775764854.694204:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a260. 00010000:00000010:3.1:1775764854.703215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da000. 00000080:00000010:3.1:1775764854.703223:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bf20. 00010000:00000010:3.1:1775764854.734446:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902400. 00010000:00000010:2.1:1775764854.735204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994200030800. 00010000:00000010:3.1:1775764854.756228:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da000. 00010000:00000010:0.1:1775764854.760348:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dba00. 00010000:00000010:3.1:1775764854.779589:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db400. 00010000:00000010:3.1:1775764854.813256:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994200030800. 00010000:00000010:3.1:1775764854.862947:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078e9300. 00010000:00000010:3.1:1775764854.862958:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898c00. 00010000:00000010:3.1:1775764854.862963:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942420d9f00. 00010000:00000010:3.1:1775764854.862966:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079ca800. 00010000:00000010:2.1:1775764854.866247:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078a1900. 00010000:00000010:2.1:1775764854.866265:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903e00. 00010000:00000010:2.1:1775764854.870280:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942069a5f00. 00010000:00000010:2.1:1775764854.870292:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903c00. 00000080:00000010:3.1:1775764854.905417:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2320. 00010000:00000010:3.1:1775764854.905435:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898c00. 00010000:00000010:2.1:1775764854.909202:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078a1100. 00010000:00000010:2.1:1775764854.909213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db400. 00000080:00000010:0.1:1775764854.909288:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db25c0. 00010000:00000010:0.1:1775764854.909305:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942420d9200. 00010000:00000010:0.1:1775764854.909309:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079ca400. 00010000:00000010:3.1:1775764854.918339:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d600. 00000080:00000010:0.1:1775764854.929203:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8b80. 00000080:00000010:0.1:1775764854.929239:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994218993180. 00000080:00000010:3.1:1775764854.933224:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6be40. 00000080:00000010:3.1:1775764854.933263:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff99422762dac0. 00010000:00000010:2.1:1775764854.947371:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078e9800. 00010000:00000010:2.1:1775764854.947382:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898400. 00010000:00000010:2.1:1775764854.971212:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420b307300. 00010000:00000010:2.1:1775764854.971223:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db800. 00010000:00000010:2.1:1775764854.981986:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898600. 00010000:00000010:2.1:1775764854.995648:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994200030800. 00010000:00000010:3.1:1775764855.010193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6600. 00000080:00000010:2.1:1775764855.010446:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bf20. 00010000:00000010:2.1:1775764855.022248:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7e00. 00000080:00000010:2.1:1775764855.022255:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db28e0. 00000080:00000010:0.1:1775764855.022317:0:2392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db22a0. 00010000:00000010:1.1:1775764855.027000:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898400. 00010000:00000010:2.1:1775764855.039955:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6400. 00000080:00000010:2.1:1775764855.039979:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2300. 00010000:00000010:2.1:1775764855.051295:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898600. 00010000:00000010:2.1:1775764855.051351:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078e9200. 00010000:00000010:2.1:1775764855.051357:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899a00. 00010000:00000010:2.1:1775764855.051361:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6a00. 00010000:00000010:3.1:1775764855.072343:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da000. 00010000:00000010:3.1:1775764855.072384:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207eef900. 00010000:00000010:3.1:1775764855.072391:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7c00. 00010000:00000010:3.1:1775764855.072394:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7000. 00000080:00000010:2.1:1775764855.080219:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c81e0. 00000080:00000010:2.1:1775764855.080243:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db22c0. 00000080:00000010:3.1:1775764855.084241:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6be40. 00000080:00000010:3.1:1775764855.084270:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c83a0. 00010000:00000010:2.1:1775764855.105679:0:2402:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207eef700. 00010000:00000010:2.1:1775764855.105700:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6a00. 00010000:00000010:2.1:1775764855.128213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898600. 00010000:00000010:3.1:1775764855.135738:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994218f6fe00. 00000080:00000010:3.1:1775764855.135747:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a340. 00010000:00000010:2.1:1775764855.148700:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899a00. 00010000:00000010:2.1:1775764855.171445:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898400. 00000080:00000010:2.1:1775764855.171454:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8060. 00010000:00000010:1.1:1775764855.193811:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7e00. 00010000:00000010:0.1:1775764855.222480:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dbc00. 00010000:00000010:2.1:1775764855.223368:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7000. 00010000:00000010:2.1:1775764855.241506:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7c00. 00000080:00000010:2.1:1775764855.242667:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db21e0. 00010000:00000010:2.1:1775764855.242676:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da000. 00010000:00000010:3.1:1775764855.247613:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6600. 00010000:00000010:2.1:1775764855.296236:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420b307300. 00010000:00000010:2.1:1775764855.296250:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7000. 00010000:00000010:3.1:1775764855.299441:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da000. 00000080:00000010:3.1:1775764855.299499:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2160. 00010000:00000010:3.1:1775764855.299507:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6400. 00000080:00000010:2.1:1775764855.300270:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2780. 00000080:00000010:2.1:1775764855.300291:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d8840. 00010000:00000010:2.1:1775764855.336509:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943800. 00010000:00000010:2.1:1775764855.336528:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dbc00. 00000080:00000010:2.1:1775764855.352294:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8c40. 00010000:00000010:3.1:1775764855.352471:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899a00. 00000080:00000010:0.1:1775764855.361357:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bc60. 00000080:00000010:0.1:1775764855.361381:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d98c0. 00010000:00000010:0.1:1775764855.368567:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898400. 00000080:00000010:0.1:1775764855.368576:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8320. 00010000:00000010:1.1:1775764855.385313:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7c00. 00010000:00000010:0.1:1775764855.400490:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898400. 00000080:00000010:0.1:1775764855.400588:0:12:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2780. 00010000:00000010:0.1:1775764855.400598:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7000. 00010000:00000010:3.1:1775764855.423984:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d400. 00000080:00000010:3.1:1775764855.424050:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8d80. 00010000:00000010:0.1:1775764855.424529:0:6158:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899a00. 00010000:00000010:3.1:1775764855.432997:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898600. 00010000:00000010:2.1:1775764855.433215:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d600. 00010000:00000010:3.1:1775764855.482360:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798dc00. 00010000:00000010:2.1:1775764855.536200:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7c00. 00000080:00000010:2.1:1775764855.536208:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2040. 00010000:00000010:2.1:1775764855.563236:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6600. 00000080:00000010:2.1:1775764855.563249:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2500. 00010000:00000010:3.1:1775764855.576207:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078e9200. 00010000:00000010:3.1:1775764855.576216:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898c00. 00010000:00000010:2.1:1775764855.586254:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994218f6ec00. 00000080:00000010:1.1:1775764855.600217:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8320. 00010000:00000010:3.1:1775764855.600455:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943100. 00010000:00000010:3.1:1775764855.600464:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da000. 00010000:00000010:3.1:1775764855.605280:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6600. 00010000:00000010:3.1:1775764855.605288:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994218f6ec00. 00010000:00000010:2.1:1775764855.613321:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7e00. 00010000:00000010:2.1:1775764855.626380:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942420d9d00. 00010000:00000010:2.1:1775764855.626392:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994218f6fe00. 00010000:00000010:2.1:1775764855.626396:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6400. 00000080:00000010:3.1:1775764855.629250:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2040. 00000080:00000010:3.1:1775764855.629265:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6ad60. 00000080:00000010:2.1:1775764855.629551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c83c0. 00010000:00000010:0.1:1775764855.633612:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6600. 00010000:00000010:2.1:1775764855.638228:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207eeff00. 00010000:00000010:2.1:1775764855.638235:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7c00. 00010000:00000010:2.1:1775764855.638238:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db400. 00010000:00000010:3.1:1775764855.640696:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7e00. 00010000:00000010:1.1:1775764855.642224:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943100. 00010000:00000010:1.1:1775764855.642232:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da000. 00000080:00000010:3.1:1775764855.648530:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db27a0. 00000080:00000010:3.1:1775764855.648543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2500. 00010000:00000010:3.1:1775764855.660611:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994200030800. 00000080:00000010:2.1:1775764855.674406:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c81c0. 00010000:00000010:0.1:1775764855.681202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c200. 00010000:00000010:3.1:1775764855.681962:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207eefe00. 00010000:00000010:3.1:1775764855.681970:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6600. 00010000:00000010:2.1:1775764855.727447:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6400. 00010000:00000010:3.1:1775764855.729257:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6600. 00010000:00000010:2.1:1775764855.741234:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7e00. 00010000:00000010:1.1:1775764855.749098:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994218f6fe00. 00000080:00000010:1.1:1775764855.749114:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6aa40. 00010000:00000010:2.1:1775764855.781309:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994201483600. 00010000:00000010:2.1:1775764855.817066:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994201483600. 00010000:00000010:2.1:1775764855.825347:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994218f6e800. 00010000:00000010:3.1:1775764855.849671:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994201483600. 00010000:00000010:2.1:1775764855.880206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994218f6ee00. 00000080:00000010:0.1:1775764855.880369:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a880. 00010000:00000010:0.1:1775764855.905223:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207eeff00. 00010000:00000010:0.1:1775764855.905231:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7c00. 00000080:00000010:0.1:1775764855.919240:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b360. 00000080:00000010:0.1:1775764855.919279:0:2393:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff99422762b9c0. 00010000:00000010:3.1:1775764855.942069:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207eefb00. 00010000:00000010:3.1:1775764855.942082:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994201482600. 00000080:00000010:2.1:1775764855.963953:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db26c0. 00010000:00000010:1.1:1775764855.964197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994201483600. 00010000:00000010:3.1:1775764855.990243:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798dc00. 00010000:00000010:2.1:1775764855.990371:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943b00. 00010000:00000010:2.1:1775764855.990378:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c200. 00000080:00000010:2.1:1775764855.995639:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bec0. 00000080:00000010:2.1:1775764855.995676:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d8000. 00000080:00000010:0.1:1775764855.996251:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2fc0. 00010000:00000010:2.1:1775764856.007253:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994201483600. 00010000:00000010:3.1:1775764856.009368:0:32:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207eef000. 00010000:00000010:3.1:1775764856.009384:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994201483e00. 00010000:00000010:2.1:1775764856.023243:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994201482600. 00000080:00000010:2.1:1775764856.023255:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db26c0. 00010000:00000010:2.1:1775764856.040302:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994218f6e400. 00000080:00000010:2.1:1775764856.066619:0:13:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2f60. 00010000:00000010:3.1:1775764856.087832:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db400. 00010000:00000010:3.1:1775764856.087879:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943100. 00010000:00000010:3.1:1775764856.087884:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da000. 00010000:00000010:3.1:1775764856.087886:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d200. 00010000:00000010:0.1:1775764856.096220:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994201482600. 00000080:00000010:3.1:1775764856.107331:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b960. 00000080:00000010:3.1:1775764856.107345:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8060. 00010000:00000010:2.1:1775764856.107868:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994218f6fe00. 00000080:00000010:2.1:1775764856.107875:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a440. 00010000:00000010:3.1:1775764856.127329:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da000. 00010000:00000010:0.1:1775764856.128359:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994201482600. 00010000:00000010:0.1:1775764856.128405:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942420d9000. 00010000:00000010:0.1:1775764856.128411:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994218f6ec00. 00010000:00000010:0.1:1775764856.128414:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994201483600. 00010000:00000010:2.1:1775764856.131333:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994201483e00. 00000080:00000010:2.1:1775764856.131355:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2b00. 00000080:00000010:0.1:1775764856.132208:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2f60. 00000080:00000010:0.1:1775764856.132223:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db26c0. 00010000:00000010:0.1:1775764856.172197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7e00. 00010000:00000010:2.1:1775764856.197659:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6600. 00000080:00000010:2.1:1775764856.197671:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bf00. 00010000:00000010:3.1:1775764856.206370:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6400. 00010000:00000010:1.1:1775764856.222538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7c00. 00010000:00000010:2.1:1775764856.233188:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6600. 00010000:00000010:2.1:1775764856.267334:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7e00. 00000080:00000010:0.1:1775764856.288057:0:14454:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275da940. 00010000:00000010:2.1:1775764856.288663:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943800. 00010000:00000010:2.1:1775764856.288753:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994218f6fe00. 00000080:00000010:1.1:1775764856.289062:0:20:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2b00. 00010000:00000010:2.1:1775764856.298029:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207eefb00. 00010000:00000010:2.1:1775764856.298038:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7c00. 00010000:00000010:3.1:1775764856.298214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798dc00. 00000080:00000010:3.1:1775764856.298222:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8260. 00000080:00000010:2.1:1775764856.307250:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2440. 00000080:00000010:2.1:1775764856.307288:0:14453:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d6b40. 00010000:00000010:2.1:1775764856.328583:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7e00. 00000080:00000010:2.1:1775764856.328591:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db23c0. 00010000:00000010:3.1:1775764856.340203:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6400. 00010000:00000010:2.1:1775764856.342572:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db400. 00010000:00000010:2.1:1775764856.359003:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6400. 00010000:00000010:2.1:1775764856.370280:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7c00. 00010000:00000010:2.1:1775764856.375347:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994200030800. 00010000:00000010:3.1:1775764856.423325:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994218f6ec00. 00010000:00000010:3.1:1775764856.466183:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207976a00. 00010000:00000010:3.1:1775764856.466191:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971000. 00010000:00000010:0.1:1775764856.472191:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798da00. 00000080:00000010:0.1:1775764856.472199:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8e00. 00010000:00000010:0.1:1775764856.501651:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d200. 00010000:00000010:0.1:1775764856.501737:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798dc00. 00010000:00000010:2.1:1775764856.510706:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994218f6fe00. 00000080:00000010:2.1:1775764856.510716:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a260. 00010000:00000010:0.1:1775764856.548325:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942420d9000. 00010000:00000010:0.1:1775764856.548341:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798da00. 00010000:00000010:2.1:1775764856.555827:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6400. 00010000:00000010:2.1:1775764856.555877:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7c00. 00000080:00000010:0.1:1775764856.573220:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c83c0. 00000080:00000010:0.1:1775764856.583214:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227604200. 00010000:00000010:2.1:1775764856.588072:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994201483e00. 00000080:00000010:2.1:1775764856.588081:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8dc0. 00010000:00000010:2.1:1775764856.599799:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943800. 00010000:00000010:2.1:1775764856.599810:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207938200. 00000080:00000010:2.1:1775764856.601330:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2b20. 00000080:00000010:2.1:1775764856.601358:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d3180. 00010000:00000010:3.1:1775764856.609341:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6400. 00000080:00000010:3.1:1775764856.609355:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db26c0. 00010000:00000010:0.1:1775764856.613296:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994201483600. 00010000:00000010:1.1:1775764856.623240:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7c00. 00010000:00000010:3.1:1775764856.650804:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903e00. 00010000:00000010:2.1:1775764856.657258:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903c00. 00010000:00000010:3.1:1775764856.690232:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994218f6e400. 00010000:00000010:3.1:1775764856.706609:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6400. 00010000:00000010:3.1:1775764856.796939:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903e00. 00000080:00000010:3.1:1775764856.796965:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bc40. 00010000:00000010:2.1:1775764856.830833:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994201482600. 00000080:00000010:2.1:1775764856.830849:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c88c0. 00010000:00000010:2.1:1775764856.862613:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994218f6e400. 00010000:00000010:2.1:1775764856.867853:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994201483e00. 00010000:00000010:0.1:1775764856.884229:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d200. 00010000:00000010:0.1:1775764856.884287:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d89800. 00010000:00000010:3.1:1775764856.912531:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d89c00. 00010000:00000010:2.1:1775764856.946178:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d200. 00010000:00000010:3.1:1775764856.987817:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078a1a00. 00010000:00000010:3.1:1775764856.987835:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899200. 00010000:00000010:3.1:1775764856.995199:0:13:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943500. 00010000:00000010:3.1:1775764856.995220:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db800. 00010000:00000010:3.1:1775764856.995226:0:13:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078e9800. 00010000:00000010:3.1:1775764856.995233:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994218f6f000. 00010000:00000010:0.1:1775764857.004486:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078e9500. 00010000:00000010:0.1:1775764857.004520:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7000. 00010000:00000010:0.1:1775764857.004526:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943d00. 00010000:00000010:0.1:1775764857.004529:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db600. 00010000:00000010:0.1:1775764857.014292:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207931000. 00010000:00000010:0.1:1775764857.014302:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8200. 00010000:00000010:2.1:1775764857.062303:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943100. 00010000:00000010:2.1:1775764857.062314:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dbc00. 00010000:00000010:1.1:1775764857.068420:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207eef000. 00010000:00000010:1.1:1775764857.068430:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d400. 00000080:00000010:2.1:1775764857.086193:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b620. 00000080:00000010:2.1:1775764857.086247:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d0840. 00000080:00000010:2.1:1775764857.086253:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c81c0. 00000080:00000010:2.1:1775764857.090226:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d0000. 00010000:00000010:3.1:1775764857.093319:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078a1400. 00010000:00000010:3.1:1775764857.093337:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d88000. 00010000:00000010:3.1:1775764857.103222:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d88200. 00010000:00000010:2.1:1775764857.107201:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078e9000. 00010000:00000010:2.1:1775764857.107211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798dc00. 00000080:00000010:3.1:1775764857.117268:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a760. 00010000:00000010:0.1:1775764857.131416:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903e00. 00010000:00000010:0.1:1775764857.143226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902400. 00000080:00000010:0.1:1775764857.143246:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b600. 00010000:00000010:3.1:1775764857.145943:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d88e00. 00010000:00000010:2.1:1775764857.185241:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902400. 00010000:00000010:2.1:1775764857.185298:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207976b00. 00010000:00000010:2.1:1775764857.185303:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916400. 00010000:00000010:2.1:1775764857.185306:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d400. 00000080:00000010:2.1:1775764857.192545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c88c0. 00000080:00000010:2.1:1775764857.211573:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b620. 00000080:00000010:2.1:1775764857.211586:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bc40. 00010000:00000010:3.1:1775764857.226086:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d88c00. 00010000:00000010:2.1:1775764857.241849:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d89000. 00000080:00000010:2.1:1775764857.241861:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2940. 00010000:00000010:3.1:1775764857.251536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798da00. 00010000:00000010:0.1:1775764857.278268:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903e00. 00010000:00000010:0.1:1775764857.278325:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207eef000. 00010000:00000010:0.1:1775764857.278331:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917800. 00010000:00000010:0.1:1775764857.278336:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d88400. 00010000:00000010:2.1:1775764857.283143:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d89000. 00000080:00000010:2.1:1775764857.283178:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2fc0. 00000080:00000010:2.1:1775764857.286222:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b960. 00000080:00000010:2.1:1775764857.286255:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2b20. 00010000:00000010:0.1:1775764857.338769:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d400. 00010000:00000010:3.1:1775764857.365586:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903c00. 00000080:00000010:3.1:1775764857.365608:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bd80. 00010000:00000010:2.1:1775764857.376016:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d89000. 00010000:00000010:3.1:1775764857.408781:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d88e00. 00010000:00000010:3.1:1775764857.424855:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00010000:00000010:2.1:1775764857.429199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d89000. 00010000:00000010:3.1:1775764857.438370:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798dc00. 00010000:00000010:0.1:1775764857.466204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d400. 00010000:00000010:3.1:1775764857.476201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899800. 00010000:00000010:3.1:1775764857.476209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916c00. 00010000:00000010:3.1:1775764857.488200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d89000. 00010000:00000010:2.1:1775764857.492217:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0400. 00010000:00000010:3.1:1775764857.503192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917a00. 00010000:00000010:2.1:1775764857.514788:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916a00. 00010000:00000010:3.1:1775764857.532274:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1800. 00010000:00000010:3.1:1775764857.564347:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917400. 00010000:00000010:3.1:1775764857.596851:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d89c00. 00010000:00000010:3.1:1775764857.613191:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0600. 00010000:00000010:3.1:1775764857.613201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917a00. 00010000:00000010:0.1:1775764857.619562:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d89800. 00010000:00000010:2.1:1775764857.684464:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da800. 00010000:00000010:2.1:1775764857.729230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d89c00. 00010000:00000010:3.1:1775764857.774359:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917200. 00000080:00000010:3.1:1775764857.774368:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6abe0. 00010000:00000010:3.1:1775764857.802038:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d89c00. 00000080:00000010:1.1:1775764857.818425:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8300. 00010000:00000010:0.1:1775764857.824738:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d200. 00010000:00000010:0.1:1775764857.847765:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d88e00. 00010000:00000010:0.1:1775764857.847812:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903c00. 00010000:00000010:3.1:1775764857.879582:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207ce3000. 00010000:00000010:3.1:1775764857.879593:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798da00. 00010000:00000010:0.1:1775764857.889061:0:6159:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d200. 00010000:00000010:3.1:1775764857.897133:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902400. 00010000:00000010:3.1:1775764857.897195:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916a00. 00010000:00000010:3.1:1775764857.904195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d89c00. 00010000:00000010:2.1:1775764857.939344:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d88400. 00010000:00000010:2.1:1775764857.939357:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207ce3100. 00010000:00000010:2.1:1775764857.939362:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903c00. 00010000:00000010:2.1:1775764857.972746:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d89c00. 00010000:00000010:3.1:1775764858.143192:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d88e00. 00010000:00000010:2.1:1775764858.655338:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d88400. 00000080:00000010:2.1:1775764858.655370:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2e40. 00010000:00000010:3.1:1775764858.733415:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dbc00. 00010000:00000010:3.1:1775764858.733473:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d89c00. 00000080:00000010:3.1:1775764858.806760:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db28e0. 00000080:00000010:3.1:1775764858.806795:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2500. 00010000:00000010:3.1:1775764858.819199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dbc00. 00000080:00000010:3.1:1775764858.819212:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b360. 00000080:00000010:2.1:1775764858.831422:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db21e0. 00010000:00000010:2.1:1775764858.831440:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207976d00. 00010000:00000010:2.1:1775764858.831445:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da800. 00000080:00000010:2.1:1775764858.847408:0:13:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b280. 00000080:00000010:2.1:1775764858.847432:0:13:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff99422762b180. 00000080:00000010:0.1:1775764858.852818:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2e40. 00010000:00000010:3.1:1775764858.853180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dbc00. 00010000:00000010:2.1:1775764858.853234:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d88400. 00010000:00000010:0.1:1775764858.868841:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db600. 00010000:00000010:3.1:1775764858.900234:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dbc00. 00010000:00000010:2.1:1775764858.933177:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d400. 00000080:00000010:1.1:1775764858.945204:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2300. 00010000:00000010:3.1:1775764858.946712:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d89c00. 00000080:00000010:0.1:1775764858.964202:0:1026:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8000. 00000080:00000010:2.1:1775764858.981202:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a260. 00000080:00000010:2.1:1775764858.996207:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2780. 00010000:00000010:2.1:1775764858.996226:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420788e900. 00010000:00000010:2.1:1775764858.996233:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798da00. 00000080:00000010:2.1:1775764859.004839:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b360. 00000080:00000010:2.1:1775764859.004879:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942189a8000. 00000080:00000010:2.1:1775764859.007222:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2160. 00010000:00000010:3.1:1775764859.007641:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dbc00. 00000080:00000010:3.1:1775764859.007661:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bc60. 00010000:00000010:0.1:1775764859.008194:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793a200. 00010000:00000010:1.1:1775764859.049281:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793a200. 00010000:00000010:1.1:1775764859.081551:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db800. 00010000:00000010:3.1:1775764859.083570:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dbc00. 00010000:00000010:3.1:1775764859.084848:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793be00. 00010000:00000010:3.1:1775764859.132020:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793a200. 00010000:00000010:2.1:1775764859.151222:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943d00. 00010000:00000010:2.1:1775764859.151231:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917c00. 00000080:00000010:3.1:1775764859.159840:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2160. 00000080:00000010:3.1:1775764859.159862:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275dd280. 00010000:00000010:1.1:1775764859.160012:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dbc00. 00000080:00000010:1.1:1775764859.160020:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b400. 00010000:00000010:2.1:1775764859.163207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793a200. 00000080:00000010:2.1:1775764859.163217:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db26a0. 00010000:00000010:2.1:1775764859.192598:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793a200. 00010000:00000010:2.1:1775764859.206051:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898600. 00010000:00000010:0.1:1775764859.209494:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898400. 00010000:00000010:0.1:1775764859.209550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793be00. 00010000:00000010:3.1:1775764859.235250:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d400. 00010000:00000010:2.1:1775764859.244987:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207976d00. 00010000:00000010:2.1:1775764859.244999:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899a00. 00000080:00000010:2.1:1775764859.274278:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8980. 00000080:00000010:2.1:1775764859.274301:0:14453:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227628840. 00010000:00000010:0.1:1775764859.275856:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898600. 00000080:00000010:1.1:1775764859.278365:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c81e0. 00010000:00000010:2.1:1775764859.281745:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dbc00. 00000080:00000010:3.1:1775764859.296798:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2720. 00010000:00000010:3.1:1775764859.329079:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917c00. 00010000:00000010:3.1:1775764859.329136:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917200. 00010000:00000010:2.1:1775764859.351425:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793a200. 00010000:00000010:2.1:1775764859.351492:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db800. 00010000:00000010:0.1:1775764859.380234:0:2392:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943900. 00010000:00000010:0.1:1775764859.380266:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dbc00. 00010000:00000010:3.1:1775764859.381294:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971000. 00000080:00000010:1.1:1775764859.406283:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6ae40. 00000080:00000010:1.1:1775764859.406307:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275dd280. 00010000:00000010:1.1:1775764859.426215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916c00. 00000080:00000010:1.1:1775764859.426260:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6aa00. 00010000:00000010:2.1:1775764859.433016:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971a00. 00000080:00000010:2.1:1775764859.433038:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2540. 00010000:00000010:2.1:1775764859.470324:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971a00. 00010000:00000010:2.1:1775764859.483396:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971600. 00010000:00000010:1.1:1775764859.486716:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917a00. 00010000:00000010:1.1:1775764859.486904:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916200. 00010000:00000010:2.1:1775764859.516192:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207c70100. 00010000:00000010:2.1:1775764859.516200:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970e00. 00000080:00000010:3.1:1775764859.529334:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2540. 00000080:00000010:3.1:1775764859.529904:0:32:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d0840. 00010000:00000010:3.1:1775764859.542105:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916200. 00010000:00000010:2.1:1775764859.542249:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971600. 00000080:00000010:2.1:1775764859.542259:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2240. 00010000:00000010:3.1:1775764859.594458:0:31:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dbc00. 00010000:00000010:2.1:1775764859.609087:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970e00. 00000080:00000010:2.1:1775764859.614375:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2100. 00010000:00000010:2.1:1775764859.621231:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971800. 00010000:00000010:2.1:1775764859.625132:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916a00. 00010000:00000010:2.1:1775764859.625148:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207ce3900. 00010000:00000010:2.1:1775764859.625168:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d89800. 00010000:00000010:0.1:1775764859.643531:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970000. 00000080:00000010:2.1:1775764859.660253:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2320. 00000080:00000010:2.1:1775764859.660287:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942189aa100. 00000080:00000010:3.1:1775764859.716686:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bce0. 00010000:00000010:3.1:1775764859.716719:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971800. 00010000:00000010:3.1:1775764859.740080:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d88200. 00010000:00000010:3.1:1775764859.749198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970e00. 00010000:00000010:0.1:1775764859.767554:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d400. 00010000:00000010:2.1:1775764859.785330:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994201482600. 00010000:00000010:2.1:1775764859.785341:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420788ef00. 00010000:00000010:2.1:1775764859.785345:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994201483e00. 00000080:00000010:2.1:1775764859.785351:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b7a0. 00000080:00000010:0.1:1775764859.809806:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942189af380. 00010000:00000010:1.1:1775764859.810301:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917200. 00010000:00000010:0.1:1775764859.844018:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798dc00. 00010000:00000010:3.1:1775764859.844775:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899a00. 00010000:00000010:3.1:1775764859.873386:0:6169:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970200. 00010000:00000010:2.1:1775764859.897959:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917000. 00010000:00000010:1.1:1775764859.901043:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899200. 00010000:00000010:3.1:1775764859.930847:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970e00. 00010000:00000010:3.1:1775764859.930869:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917800. 00010000:00000010:1.1:1775764859.959296:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970000. 00010000:00000010:1.1:1775764859.959315:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917200. 00010000:00000010:3.1:1775764860.000779:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d88000. 00010000:00000010:0.1:1775764860.004316:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899a00. 00000080:00000010:0.1:1775764860.004329:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8860. 00010000:00000010:3.1:1775764860.015364:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971000. 00010000:00000010:2.1:1775764860.078734:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971000. 00010000:00000010:2.1:1775764860.078785:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00010000:00000010:0.1:1775764860.093396:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917400. 00010000:00000010:3.1:1775764860.120188:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d89000. 00000080:00000010:0.1:1775764860.129368:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2320. 00010000:00000010:2.1:1775764860.139243:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916200. 00000080:00000010:0.1:1775764860.140450:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a420. 00000080:00000010:0.1:1775764860.145273:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8c40. 00010000:00000010:0.1:1775764860.145298:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420788ef00. 00010000:00000010:0.1:1775764860.145305:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970000. 00000080:00000010:0.1:1775764860.151304:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2700. 00010000:00000010:2.1:1775764860.172249:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970200. 00000080:00000010:2.1:1775764860.172265:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2660. 00000080:00000010:0.1:1775764860.173387:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942189af380. 00010000:00000010:3.1:1775764860.208744:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917800. 00010000:00000010:0.1:1775764860.221245:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917000. 00010000:00000010:1.1:1775764860.221947:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916a00. 00010000:00000010:3.1:1775764860.224961:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970200. 00010000:00000010:3.1:1775764860.228450:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970000. 00010000:00000010:2.1:1775764860.229913:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994201483e00. 00000080:00000010:2.1:1775764860.229927:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2d80. 00000080:00000010:2.1:1775764860.275249:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8000. 00010000:00000010:1.1:1775764860.278922:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917a00. 00010000:00000010:1.1:1775764860.278980:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994201482600. 00000080:00000010:3.1:1775764860.279257:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2100. 00000080:00000010:3.1:1775764860.299234:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a6a0. 00010000:00000010:3.1:1775764860.299256:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943900. 00010000:00000010:3.1:1775764860.299263:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798dc00. 00000080:00000010:3.1:1775764860.318489:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2660. 00000080:00000010:3.1:1775764860.335386:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275ddac0. 00010000:00000010:1.1:1775764860.373124:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0e00. 00010000:00000010:3.1:1775764860.376099:0:2403:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994218fe6800. 00010000:00000010:3.1:1775764860.376119:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917000. 00000080:00000010:3.1:1775764860.376125:0:2403:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a780. 00000080:00000010:3.1:1775764860.376148:0:2403:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942189aa100. 00010000:00000010:3.1:1775764860.402412:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917800. 00010000:00000010:0.1:1775764860.411150:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917a00. 00010000:00000010:0.1:1775764860.411170:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994201482600. 00010000:00000010:1.1:1775764860.425302:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793a200. 00010000:00000010:3.1:1775764860.433258:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917c00. 00010000:00000010:2.1:1775764860.443143:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789c200. 00000080:00000010:2.1:1775764860.443169:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2b80. 00010000:00000010:0.1:1775764860.447130:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0400. 00000080:00000010:0.1:1775764860.457873:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8260. 00010000:00000010:2.1:1775764860.498198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789c200. 00010000:00000010:2.1:1775764860.498245:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789cc00. 00010000:00000010:0.1:1775764860.507719:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917800. 00010000:00000010:0.1:1775764860.507951:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0e00. 00010000:00000010:1.1:1775764860.561216:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078e9800. 00010000:00000010:1.1:1775764860.561228:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1800. 00010000:00000010:2.1:1775764860.566705:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207c60600. 00010000:00000010:2.1:1775764860.566714:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789da00. 00000080:00000010:3.1:1775764860.580136:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a580. 00000080:00000010:2.1:1775764860.580553:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2b80. 00000080:00000010:2.1:1775764860.580589:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942189aa100. 00000080:00000010:2.1:1775764860.595778:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227628840. 00010000:00000010:1.1:1775764860.606633:0:1026:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789cc00. 00010000:00000010:0.1:1775764860.616402:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078a1e00. 00010000:00000010:0.1:1775764860.616419:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80e00. 00010000:00000010:2.1:1775764860.617748:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1600. 00010000:00000010:3.1:1775764860.650372:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0400. 00000080:00000010:3.1:1775764860.650384:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c88c0. 00010000:00000010:2.1:1775764860.665117:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789da00. 00010000:00000010:1.1:1775764860.689286:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1800. 00010000:00000010:1.1:1775764860.689330:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798dc00. 00010000:00000010:0.1:1775764860.698385:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917200. 00010000:00000010:2.1:1775764860.708112:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789da00. 00000080:00000010:2.1:1775764860.708122:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2d80. 00010000:00000010:2.1:1775764860.765181:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0400. 00010000:00000010:0.1:1775764860.765242:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078e9d00. 00010000:00000010:0.1:1775764860.765249:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917c00. 00000080:00000010:0.1:1775764860.765255:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8e80. 00000080:00000010:0.1:1775764860.765272:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff99422762b180. 00010000:00000010:2.1:1775764860.776236:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789c800. 00010000:00000010:2.1:1775764860.790034:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078e9700. 00010000:00000010:2.1:1775764860.790050:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917e00. 00010000:00000010:0.1:1775764860.816609:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0e00. 00000080:00000010:2.1:1775764860.843884:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c17040. 00010000:00000010:2.1:1775764860.843919:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207c60b00. 00010000:00000010:2.1:1775764860.843927:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789da00. 00000080:00000010:2.1:1775764860.843933:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8b60. 00000080:00000010:2.1:1775764860.843951:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942189aa100. 00010000:00000010:3.1:1775764860.847770:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994218fe6b00. 00010000:00000010:3.1:1775764860.847779:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793d200. 00010000:00000010:2.1:1775764860.869931:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fca00. 00000080:00000010:2.1:1775764860.869948:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2820. 00010000:00000010:2.1:1775764860.879225:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0e00. 00010000:00000010:0.1:1775764860.887250:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0600. 00000080:00000010:0.1:1775764860.887271:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8980. 00010000:00000010:2.1:1775764860.938769:0:1026:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0600. 00010000:00000010:2.1:1775764860.942640:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc600. 00010000:00000010:0.1:1775764860.966083:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917e00. 00010000:00000010:2.1:1775764860.968364:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0400. 00010000:00000010:0.1:1775764860.981254:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0e00. 00010000:00000010:1.1:1775764861.010765:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798dc00. 00000080:00000010:2.1:1775764861.026275:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8f60. 00010000:00000010:3.1:1775764861.033208:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd200. 00000080:00000010:2.1:1775764861.034560:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207c17aa0. 00010000:00000010:2.1:1775764861.043068:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078a1e00. 00010000:00000010:2.1:1775764861.043093:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994203fb1200. 00000080:00000010:2.1:1775764861.077410:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6aea0. 00010000:00000010:2.1:1775764861.090209:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078a1700. 00010000:00000010:2.1:1775764861.090218:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81600. 00000080:00000010:0.1:1775764861.090772:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275ddac0. 00010000:00000010:3.1:1775764861.096672:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd400. 00010000:00000010:2.1:1775764861.100248:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793d800. 00000080:00000010:2.1:1775764861.100277:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c81c0. 00010000:00000010:3.1:1775764861.165206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793d800. 00010000:00000010:3.1:1775764861.165220:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd800. 00010000:00000010:1.1:1775764861.168625:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc200. 00010000:00000010:3.1:1775764861.195961:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fce00. 00010000:00000010:2.1:1775764861.254196:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078e9f00. 00010000:00000010:2.1:1775764861.254209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793ca00. 00010000:00000010:1.1:1775764861.292236:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fde00. 00010000:00000010:3.1:1775764861.292756:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917c00. 00010000:00000010:1.1:1775764861.321989:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793d800. 00010000:00000010:2.1:1775764861.345108:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793d200. 00010000:00000010:2.1:1775764861.379236:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc200. 00010000:00000010:3.1:1775764861.383351:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917c00. 00010000:00000010:2.1:1775764861.387298:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078e9900. 00010000:00000010:2.1:1775764861.387322:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793c000. 00010000:00000010:3.1:1775764861.398257:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078e9400. 00010000:00000010:3.1:1775764861.398271:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207535c00. 00010000:00000010:0.1:1775764861.406232:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd800. 00010000:00000010:3.1:1775764861.448432:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793d800. 00010000:00000010:1.1:1775764861.497571:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd200. 00010000:00000010:2.1:1775764861.500225:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd800. 00010000:00000010:1.1:1775764861.515150:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916200. 00010000:00000010:2.1:1775764861.529412:0:14473:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078a1c00. 00010000:00000010:2.1:1775764861.529439:0:14473:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2e00. 00010000:00000010:1.1:1775764861.545091:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb9c00. 00010000:00000010:2.1:1775764861.548220:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917c00. 00010000:00000010:3.1:1775764861.581506:0:14464:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078b3a00. 00010000:00000010:3.1:1775764861.581534:0:14464:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d89600. 00010000:00000010:3.1:1775764861.587433:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fde00. 00010000:00000010:2.1:1775764861.594633:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8200. 00010000:00000010:2.1:1775764861.605206:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd200. 00010000:00000010:1.1:1775764861.640213:0:1006:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943000. 00010000:00000010:1.1:1775764861.640237:0:1006:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c000. 00000080:00000010:2.1:1775764861.640604:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6afc0. 00010000:00000010:3.1:1775764861.690830:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fde00. 00010000:00000010:0.1:1775764861.693965:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd200. 00010000:00000010:2.1:1775764861.735182:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207c67600. 00010000:00000010:2.1:1775764861.735189:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917a00. 00010000:00000010:3.1:1775764861.742235:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0400. 00010000:00000010:2.1:1775764861.796207:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0e00. 00010000:00000010:3.1:1775764861.796741:0:14493:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0600. 00010000:00000010:2.1:1775764861.871762:0:2403:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078b3800. 00010000:00000010:2.1:1775764861.871781:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d88a00. 00010000:00000010:0.1:1775764861.875347:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0e00. 00010000:00000010:2.1:1775764861.884680:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0600. 00000080:00000010:2.1:1775764861.887317:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2660. 00010000:00000010:3.1:1775764861.904207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917200. 00000080:00000010:2.1:1775764861.965296:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2680. 00010000:00000010:1.1:1775764861.984267:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0600. 00010000:00000010:2.1:1775764861.993205:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078b3a00. 00010000:00000010:2.1:1775764861.993215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793d800. 00000080:00000010:3.1:1775764861.996804:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db20a0. 00000080:00000010:3.1:1775764862.011398:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b980. 00010000:00000010:3.1:1775764862.029202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789da00. 00000080:00000010:2.1:1775764862.039186:0:13:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2820. 00000080:00000010:2.1:1775764862.063703:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b6a0. 00010000:00000010:2.1:1775764862.063742:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207eeff00. 00010000:00000010:2.1:1775764862.063750:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6600. 00010000:00000010:2.1:1775764862.073368:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207931000. 00010000:00000010:2.1:1775764862.073378:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898c00. 00010000:00000010:2.1:1775764862.073383:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943b00. 00010000:00000010:2.1:1775764862.073386:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da000. 00010000:00000010:2.1:1775764862.073390:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207eef000. 00010000:00000010:2.1:1775764862.073394:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916000. 00010000:00000010:2.1:1775764862.073397:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207976700. 00010000:00000010:2.1:1775764862.073400:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d200. 00010000:00000010:3.1:1775764862.077704:0:6169:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207eefe00. 00010000:00000010:3.1:1775764862.077713:0:6169:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dba00. 00010000:00000010:3.1:1775764862.077719:0:6169:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942069a5f00. 00010000:00000010:3.1:1775764862.077744:0:6169:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207938600. 00010000:00000010:3.1:1775764862.087187:0:2402:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942420d9d00. 00010000:00000010:3.1:1775764862.087195:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994218f6e800. 00010000:00000010:3.1:1775764862.087199:0:2402:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420788e800. 00010000:00000010:3.1:1775764862.087203:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d88400. 00010000:00000010:3.1:1775764862.087206:0:2402:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943500. 00010000:00000010:3.1:1775764862.087209:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da800. 00010000:00000010:1.1:1775764862.087324:0:2401:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207c70e00. 00010000:00000010:1.1:1775764862.087330:0:2401:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971a00. 00010000:00000010:1.1:1775764862.087335:0:2401:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207eef500. 00010000:00000010:1.1:1775764862.087340:0:2401:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d600. 00010000:00000010:1.1:1775764862.087343:0:2401:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078e9700. 00010000:00000010:1.1:1775764862.087348:0:2401:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fca00. 00010000:00000010:1.1:1775764862.087374:0:2401:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207ce3c00. 00010000:00000010:1.1:1775764862.087377:0:2401:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902400. 00010000:00000010:1.1:1775764862.087381:0:2401:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207ce3000. 00010000:00000010:1.1:1775764862.087385:0:2401:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903c00. 00010000:00000010:1.1:1775764862.087391:0:2401:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943d00. 00010000:00000010:1.1:1775764862.087394:0:2401:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970200. 00010000:00000010:3.1:1775764862.093814:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207ce3900. 00010000:00000010:3.1:1775764862.093825:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970e00. 00010000:00000010:3.1:1775764862.093830:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916200. 00010000:00000010:2.1:1775764862.100203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a6a00. 00000080:00000010:2.1:1775764862.100214:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2f80. 00010000:00000010:3.1:1775764862.108194:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a6200. 00000080:00000010:0.1:1775764862.137265:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8980. 00010000:00000010:0.1:1775764862.147268:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb9c00. 00000080:00000010:2.1:1775764862.152320:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2480. 00010000:00000010:2.1:1775764862.152339:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ef500. 00010000:00000010:2.1:1775764862.152346:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917a00. 00010000:00000010:2.1:1775764862.152351:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0e00. 00010000:00000010:3.1:1775764862.152359:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a6a00. 00000080:00000010:1.1:1775764862.152926:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2b60. 00000080:00000010:2.1:1775764862.156582:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2840. 00000080:00000010:2.1:1775764862.156609:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2820. 00000080:00000010:2.1:1775764862.156616:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c83a0. 00000080:00000010:1.1:1775764862.160215:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2e60. 00010000:00000010:2.1:1775764862.163341:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a6200. 00000080:00000010:2.1:1775764862.163350:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2c40. 00010000:00000010:2.1:1775764862.173207:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078b3700. 00010000:00000010:2.1:1775764862.173229:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917e00. 00010000:00000010:3.1:1775764862.179750:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917000. 00000080:00000010:2.1:1775764862.193849:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bf80. 00010000:00000010:1.1:1775764862.196221:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916000. 00010000:00000010:1.1:1775764862.206826:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a6a00. 00010000:00000010:3.1:1775764862.220545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a6200. 00010000:00000010:2.1:1775764862.226149:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da800. 00010000:00000010:2.1:1775764862.236107:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ef800. 00010000:00000010:2.1:1775764862.236119:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917800. 00010000:00000010:2.1:1775764862.236123:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a6400. 00010000:00000010:3.1:1775764862.246293:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916000. 00000080:00000010:2.1:1775764862.270534:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2820. 00000080:00000010:2.1:1775764862.270551:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2c40. 00010000:00000010:2.1:1775764862.276248:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da800. 00000080:00000010:2.1:1775764862.276270:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2840. 00010000:00000010:3.1:1775764862.321562:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793d800. 00010000:00000010:2.1:1775764862.323218:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da800. 00000080:00000010:2.1:1775764862.323235:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2840. 00010000:00000010:1.1:1775764862.323270:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dba00. 00010000:00000010:2.1:1775764862.413179:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917e00. 00010000:00000010:0.1:1775764862.459058:0:14:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dba00. 00010000:00000010:3.1:1775764862.495010:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793c000. 00000080:00000010:3.1:1775764862.495031:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b3e0. 00010000:00000010:2.1:1775764862.498219:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dba00. 00010000:00000010:2.1:1775764862.498285:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078b3700. 00010000:00000010:2.1:1775764862.498290:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793d800. 00010000:00000010:2.1:1775764862.498294:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da000. 00000080:00000010:2.1:1775764862.513233:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2840. 00000080:00000010:2.1:1775764862.513252:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b3c0. 00010000:00000010:3.1:1775764862.561311:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da000. 00010000:00000010:3.1:1775764862.604416:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dba00. 00010000:00000010:0.1:1775764862.625589:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8000. 00010000:00000010:2.1:1775764862.646215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da000. 00000080:00000010:2.1:1775764862.646233:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2820. 00010000:00000010:3.1:1775764862.653198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793c000. 00010000:00000010:3.1:1775764862.661606:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207c67600. 00010000:00000010:3.1:1775764862.661614:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917a00. 00010000:00000010:3.1:1775764862.661748:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793d200. 00000080:00000010:2.1:1775764862.672228:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b3e0. 00000080:00000010:2.1:1775764862.672246:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6afc0. 00010000:00000010:3.1:1775764862.686178:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793c000. 00010000:00000010:0.1:1775764862.696274:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da000. 00010000:00000010:1.1:1775764862.696331:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dba00. 00010000:00000010:0.1:1775764862.733568:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894200. 00010000:00000010:1.1:1775764862.779367:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8200. 00000080:00000010:1.1:1775764862.779382:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8f60. 00010000:00000010:3.1:1775764862.808558:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994211bdbb00. 00010000:00000010:3.1:1775764862.808568:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917e00. 00010000:00000010:2.1:1775764862.809216:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895800. 00010000:00000010:2.1:1775764862.809255:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917c00. 00000080:00000010:3.1:1775764862.821408:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b380. 00000080:00000010:3.1:1775764862.821438:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942189aa100. 00010000:00000010:0.1:1775764862.826603:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793c000. 00010000:00000010:0.1:1775764862.848217:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793d800. 00010000:00000010:3.1:1775764862.858400:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078b3a00. 00010000:00000010:3.1:1775764862.858408:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894c00. 00000080:00000010:3.1:1775764862.877357:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2680. 00010000:00000010:1.1:1775764862.877602:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb9c00. 00000080:00000010:3.1:1775764862.892954:0:14454:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275deb40. 00010000:00000010:1.1:1775764862.893535:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917a00. 00000080:00000010:1.1:1775764862.893544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a820. 00010000:00000010:1.1:1775764862.897222:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793d800. 00010000:00000010:1.1:1775764862.920226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895800. 00010000:00000010:3.1:1775764862.959441:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994211bdb900. 00010000:00000010:3.1:1775764862.959473:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c800. 00010000:00000010:2.1:1775764862.980198:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894400. 00010000:00000010:2.1:1775764862.980258:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917800. 00010000:00000010:3.1:1775764862.980723:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895200. 00010000:00000010:2.1:1775764863.004205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895800. 00000080:00000010:2.1:1775764863.017593:0:13:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2660. 00010000:00000010:3.1:1775764863.041346:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793c000. 00010000:00000010:3.1:1775764863.041516:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894c00. 00010000:00000010:2.1:1775764863.045749:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207c67600. 00010000:00000010:2.1:1775764863.045772:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894600. 00000080:00000010:2.1:1775764863.068959:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db29a0. 00000080:00000010:2.1:1775764863.068982:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff99422762dac0. 00010000:00000010:3.1:1775764863.099357:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894600. 00000080:00000010:3.1:1775764863.099366:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2660. 00000080:00000010:2.1:1775764863.118218:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2820. 00010000:00000010:2.1:1775764863.121753:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789da00. 00010000:00000010:3.1:1775764863.128011:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078b3a00. 00010000:00000010:3.1:1775764863.128024:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793d800. 00010000:00000010:0.1:1775764863.140752:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207938600. 00000080:00000010:1.1:1775764863.160913:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b380. 00000080:00000010:1.1:1775764863.160936:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275deb40. 00010000:00000010:3.1:1775764863.175779:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895800. 00010000:00000010:2.1:1775764863.192545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917200. 00000080:00000010:2.1:1775764863.192562:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a340. 00010000:00000010:2.1:1775764863.230756:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb9c00. 00010000:00000010:2.1:1775764863.230828:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894600. 00010000:00000010:1.1:1775764863.244682:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789da00. 00010000:00000010:0.1:1775764863.255331:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793d800. 00010000:00000010:3.1:1775764863.283845:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fca00. 00010000:00000010:3.1:1775764863.313035:0:32:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420950bb00. 00010000:00000010:3.1:1775764863.313066:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80400. 00010000:00000010:3.1:1775764863.322191:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894600. 00010000:00000010:0.1:1775764863.381422:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917800. 00010000:00000010:3.1:1775764863.391323:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894600. 00010000:00000010:1.1:1775764863.392086:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917a00. 00010000:00000010:0.1:1775764863.398343:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb9c00. 00000080:00000010:0.1:1775764863.398354:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c83c0. 00010000:00000010:3.1:1775764863.409201:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078b3200. 00010000:00000010:3.1:1775764863.409209:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d89200. 00010000:00000010:3.1:1775764863.453225:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207938600. 00000080:00000010:3.1:1775764863.473537:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6baa0. 00010000:00000010:3.1:1775764863.483973:0:32:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420b307300. 00010000:00000010:3.1:1775764863.483983:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8200. 00010000:00000010:3.1:1775764863.483987:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895800. 00010000:00000010:3.1:1775764863.518830:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894600. 00000080:00000010:3.1:1775764863.518846:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2520. 00010000:00000010:3.1:1775764863.518857:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd200. 00000080:00000010:0.1:1775764863.531668:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6ba60. 00010000:00000010:3.1:1775764863.540183:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207938600. 00010000:00000010:2.1:1775764863.564222:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075ca800. 00010000:00000010:0.1:1775764863.571664:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078e9000. 00010000:00000010:0.1:1775764863.571675:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075cbc00. 00010000:00000010:0.1:1775764863.571680:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793d800. 00000080:00000010:3.1:1775764863.572586:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6aba0. 00000080:00000010:0.1:1775764863.576250:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2700. 00000080:00000010:0.1:1775764863.576276:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8b60. 00010000:00000010:2.1:1775764863.585217:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078b3a00. 00010000:00000010:2.1:1775764863.585234:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894c00. 00010000:00000010:2.1:1775764863.596073:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dba00. 00000080:00000010:2.1:1775764863.596095:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8ec0. 00010000:00000010:3.1:1775764863.606338:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca0c00. 00000080:00000010:2.1:1775764863.637212:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8140. 00010000:00000010:2.1:1775764863.659372:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca0a00. 00010000:00000010:2.1:1775764863.659421:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca1400. 00010000:00000010:3.1:1775764863.669228:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca0000. 00010000:00000010:2.1:1775764863.696345:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca1400. 00010000:00000010:2.1:1775764863.696407:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994218766700. 00010000:00000010:2.1:1775764863.696412:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd200. 00010000:00000010:2.1:1775764863.696417:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075cac00. 00000080:00000010:2.1:1775764863.696428:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2540. 00000080:00000010:2.1:1775764863.696436:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2640. 00010000:00000010:3.1:1775764863.697225:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917800. 00010000:00000010:2.1:1775764863.722213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca0e00. 00000080:00000010:2.1:1775764863.722227:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2160. 00010000:00000010:3.1:1775764863.781625:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd200. 00000080:00000010:3.1:1775764863.781639:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b1e0. 00010000:00000010:2.1:1775764863.786587:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca0e00. 00010000:00000010:2.1:1775764863.786642:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca1400. 00010000:00000010:3.1:1775764863.836282:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca1400. 00010000:00000010:1.1:1775764863.856476:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca0e00. 00010000:00000010:0.1:1775764863.895287:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca0000. 00010000:00000010:2.1:1775764863.901908:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca0c00. 00000080:00000010:2.1:1775764863.901928:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2040. 00010000:00000010:1.1:1775764863.903221:0:20:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078e9000. 00010000:00000010:1.1:1775764863.903229:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420b14b600. 00010000:00000010:1.1:1775764863.903232:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075dbe00. 00000080:00000010:0.1:1775764863.913215:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2540. 00000080:00000010:0.1:1775764863.913228:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8060. 00010000:00000010:3.1:1775764863.935309:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420b14a000. 00010000:00000010:2.1:1775764863.955215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca1400. 00000080:00000010:2.1:1775764863.955238:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2300. 00010000:00000010:3.1:1775764864.034831:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca1400. 00010000:00000010:3.1:1775764864.039313:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca0c00. 00010000:00000010:2.1:1775764864.039404:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420b14ac00. 00010000:00000010:1.1:1775764864.066375:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917e00. 00010000:00000010:1.1:1775764864.071586:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207c67600. 00010000:00000010:1.1:1775764864.071600:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420b14a400. 00010000:00000010:1.1:1775764864.071605:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca0e00. 00010000:00000010:2.1:1775764864.125913:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994218766700. 00010000:00000010:2.1:1775764864.125923:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793d800. 00000080:00000010:1.1:1775764864.126911:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a9c0. 00000080:00000010:1.1:1775764864.126930:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8660. 00000080:00000010:2.1:1775764864.135193:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2300. 00000080:00000010:2.1:1775764864.135213:0:14454:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275de300. 00010000:00000010:3.1:1775764864.135752:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917e00. 00010000:00000010:3.1:1775764864.159352:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca0000. 00010000:00000010:0.1:1775764864.178851:0:6158:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420b14b600. 00000080:00000010:0.1:1775764864.178861:0:6158:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8c40. 00010000:00000010:2.1:1775764864.181548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca0e00. 00010000:00000010:3.1:1775764864.189015:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898400. 00010000:00000010:0.1:1775764864.247132:0:9316:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420b14a400. 00010000:00000010:3.1:1775764864.258076:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca0e00. 00010000:00000010:3.1:1775764864.258131:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420b14ac00. 00010000:00000010:0.1:1775764864.314204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994209db3000. 00010000:00000010:3.1:1775764864.314231:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078b3200. 00010000:00000010:3.1:1775764864.314238:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dba00. 00000080:00000010:3.1:1775764864.321254:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2b20. 00010000:00000010:2.1:1775764864.321371:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d400. 00000080:00000010:2.1:1775764864.324245:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275dc200. 00010000:00000010:3.1:1775764864.324313:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca0e00. 00010000:00000010:1.1:1775764864.328221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca1400. 00010000:00000010:1.1:1775764864.360206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca0000. 00010000:00000010:3.1:1775764864.386591:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971800. 00010000:00000010:3.1:1775764864.389459:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dba00. 00010000:00000010:2.1:1775764864.394189:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916c00. 00010000:00000010:1.1:1775764864.395354:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca1400. 00010000:00000010:0.1:1775764864.402613:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca0e00. 00010000:00000010:3.1:1775764864.408448:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899200. 00010000:00000010:2.1:1775764864.412377:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420b14a400. 00010000:00000010:3.1:1775764864.494512:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca0000. 00010000:00000010:0.1:1775764864.513023:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420b14a400. 00010000:00000010:3.1:1775764864.517955:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971000. 00010000:00000010:2.1:1775764864.528229:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca0000. 00000080:00000010:2.1:1775764864.528238:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2bc0. 00010000:00000010:2.1:1775764864.535734:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca0e00. 00010000:00000010:2.1:1775764864.569184:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793d800. 00000080:00000010:1.1:1775764864.690415:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2300. 00000080:00000010:2.1:1775764864.720503:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8e80. 00000080:00000010:2.1:1775764864.720521:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db28e0. 00000080:00000010:1.1:1775764864.749824:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b020. 00010000:00000010:3.1:1775764864.751345:0:2402:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078b3200. 00010000:00000010:3.1:1775764864.751365:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca0e00. 00000080:00000010:3.1:1775764864.751371:0:2402:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2bc0. 00000080:00000010:3.1:1775764864.751391:0:2402:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275db9c0. 00010000:00000010:3.1:1775764864.751404:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dba00. 00010000:00000010:3.1:1775764864.771574:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917a00. 00010000:00000010:2.1:1775764864.783697:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca0000. 00000080:00000010:2.1:1775764864.783722:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db28e0. 00010000:00000010:2.1:1775764864.801235:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420788e800. 00010000:00000010:2.1:1775764864.801242:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994209db3000. 00000080:00000010:2.1:1775764864.819934:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6ba40. 00000080:00000010:2.1:1775764864.819970:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275da100. 00010000:00000010:0.1:1775764864.827270:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca0e00. 00010000:00000010:3.1:1775764864.828809:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca0000. 00000080:00000010:3.1:1775764864.828830:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db28e0. 00010000:00000010:3.1:1775764864.888239:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dba00. 00010000:00000010:2.1:1775764864.909693:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca0e00. 00000080:00000010:2.1:1775764864.917418:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b880. 00010000:00000010:2.1:1775764864.917430:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994209db3000. 00000080:00000010:2.1:1775764864.928232:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2500. 00000080:00000010:3.1:1775764864.934091:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db27a0. 00000080:00000010:3.1:1775764864.968226:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bb40. 00010000:00000010:2.1:1775764864.983748:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207ce3c00. 00010000:00000010:2.1:1775764864.983758:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793d800. 00010000:00000010:0.1:1775764864.998133:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420788e800. 00010000:00000010:0.1:1775764864.998151:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1600. 00000080:00000010:3.1:1775764865.001420:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2180. 00000080:00000010:3.1:1775764865.001472:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942276018c0. 00000080:00000010:2.1:1775764865.014465:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2220. 00000080:00000010:2.1:1775764865.014496:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227618000. 00010000:00000010:2.1:1775764865.020197:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793d800. 00000080:00000010:2.1:1775764865.020207:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2500. 00010000:00000010:3.1:1775764865.021054:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dba00. 00000080:00000010:0.1:1775764865.022294:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bb40. 00010000:00000010:1.1:1775764865.066266:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994209db3000. 00010000:00000010:2.1:1775764865.092442:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793d800. 00010000:00000010:2.1:1775764865.105097:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca0e00. 00000080:00000010:2.1:1775764865.105172:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2500. 00010000:00000010:2.1:1775764865.105181:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1600. 00000080:00000010:3.1:1775764865.145377:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6aa00. 00010000:00000010:2.1:1775764865.145474:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207ce3c00. 00010000:00000010:2.1:1775764865.145486:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917c00. 00010000:00000010:2.1:1775764865.153240:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207c67600. 00010000:00000010:2.1:1775764865.153255:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075cbc00. 00000080:00000010:3.1:1775764865.156368:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227618000. 00010000:00000010:3.1:1775764865.156389:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dba00. 00000080:00000010:3.1:1775764865.156397:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b300. 00000080:00000010:2.1:1775764865.165939:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2220. 00000080:00000010:2.1:1775764865.165979:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227603180. 00010000:00000010:2.1:1775764865.190184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075cbc00. 00000080:00000010:2.1:1775764865.190200:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2500. 00010000:00000010:2.1:1775764865.213342:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca0e00. 00010000:00000010:1.1:1775764865.230320:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075cbc00. 00010000:00000010:3.1:1775764865.230414:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075ca800. 00000080:00000010:3.1:1775764865.238419:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2220. 00010000:00000010:3.1:1775764865.238427:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207938600. 00010000:00000010:2.1:1775764865.252215:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420788e800. 00010000:00000010:3.1:1775764865.252215:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943500. 00010000:00000010:3.1:1775764865.252222:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894c00. 00010000:00000010:2.1:1775764865.252223:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dba00. 00000080:00000010:2.1:1775764865.263440:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b300. 00000080:00000010:2.1:1775764865.263470:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275db9c0. 00000080:00000010:3.1:1775764865.267217:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2500. 00000080:00000010:3.1:1775764865.267248:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275da100. 00010000:00000010:3.1:1775764865.275357:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894c00. 00000080:00000010:3.1:1775764865.275366:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b940. 00010000:00000010:2.1:1775764865.288221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075ca800. 00000080:00000010:2.1:1775764865.288234:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2800. 00010000:00000010:1.1:1775764865.305211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917e00. 00010000:00000010:2.1:1775764865.310221:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075cbc00. 00010000:00000010:1.1:1775764865.323248:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917c00. 00000080:00000010:1.1:1775764865.323297:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a9a0. 00010000:00000010:1.1:1775764865.323306:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207938600. 00010000:00000010:2.1:1775764865.351221:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207eef000. 00010000:00000010:2.1:1775764865.351242:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075ca800. 00010000:00000010:3.1:1775764865.355367:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420387c000. 00010000:00000010:3.1:1775764865.355372:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917e00. 00000080:00000010:2.1:1775764865.355413:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2800. 00000080:00000010:2.1:1775764865.355441:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff99422761b9c0. 00000080:00000010:3.1:1775764865.366204:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6ad60. 00000080:00000010:3.1:1775764865.366226:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227600000. 00010000:00000010:0.1:1775764865.376554:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1000. 00000080:00000010:0.1:1775764865.376564:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8d00. 00010000:00000010:3.1:1775764865.389417:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207938600. 00000080:00000010:3.1:1775764865.389445:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b940. 00010000:00000010:2.1:1775764865.410092:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1800. 00010000:00000010:2.1:1775764865.429297:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894c00. 00010000:00000010:1.1:1775764865.461012:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943500. 00010000:00000010:1.1:1775764865.461035:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075ca800. 00000080:00000010:1.1:1775764865.474743:0:2392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2800. 00000080:00000010:1.1:1775764865.474779:0:2392:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275da100. 00010000:00000010:1.1:1775764865.479700:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917c00. 00000080:00000010:1.1:1775764865.479711:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6af00. 00010000:00000010:3.1:1775764865.492079:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207eef000. 00010000:00000010:3.1:1775764865.492091:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dba00. 00010000:00000010:1.1:1775764865.506739:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917e00. 00000080:00000010:3.1:1775764865.521409:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db26c0. 00000080:00000010:3.1:1775764865.521442:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff99422761b9c0. 00010000:00000010:2.1:1775764865.529469:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894c00. 00000080:00000010:2.1:1775764865.529480:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2220. 00010000:00000010:0.1:1775764865.553177:0:2392:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420387ca00. 00010000:00000010:0.1:1775764865.553187:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916c00. 00010000:00000010:2.1:1775764865.553805:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793d800. 00000080:00000010:2.1:1775764865.553813:0:26:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6aa40. 00000080:00000010:2.1:1775764865.553830:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227607380. 00010000:00000010:2.1:1775764865.571338:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2e00. 00000080:00000010:2.1:1775764865.571363:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2180. 00010000:00000010:3.1:1775764865.586677:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207ce3c00. 00010000:00000010:3.1:1775764865.586688:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894c00. 00010000:00000010:2.1:1775764865.590330:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2e00. 00000080:00000010:2.1:1775764865.607303:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2220. 00010000:00000010:0.1:1775764865.623226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2200. 00000080:00000010:2.1:1775764865.623242:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff99422761b9c0. 00000080:00000010:1.1:1775764865.626378:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2180. 00010000:00000010:1.1:1775764865.626388:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb3800. 00010000:00000010:2.1:1775764865.631370:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2e00. 00000080:00000010:2.1:1775764865.631391:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db28e0. 00010000:00000010:2.1:1775764865.674311:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2a00. 00010000:00000010:2.1:1775764865.683389:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966100. 00010000:00000010:2.1:1775764865.683405:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917e00. 00000080:00000010:2.1:1775764865.687962:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2e40. 00000080:00000010:2.1:1775764865.687989:0:14454:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275dc200. 00010000:00000010:2.1:1775764865.695209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2800. 00000080:00000010:2.1:1775764865.695222:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2220. 00010000:00000010:1.1:1775764865.705278:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420387cc00. 00010000:00000010:1.1:1775764865.705294:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2e00. 00000080:00000010:1.1:1775764865.712536:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db28e0. 00000080:00000010:1.1:1775764865.712560:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227607380. 00010000:00000010:2.1:1775764865.717279:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075ca800. 00000080:00000010:2.1:1775764865.717290:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a340. 00010000:00000010:0.1:1775764865.732191:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0200. 00010000:00000010:2.1:1775764865.752472:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1600. 00010000:00000010:3.1:1775764865.759106:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917e00. 00000080:00000010:3.1:1775764865.759196:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c83c0. 00000080:00000010:3.1:1775764865.759204:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c83a0. 00010000:00000010:3.1:1775764865.759209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2a00. 00010000:00000010:2.1:1775764865.761124:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2800. 00010000:00000010:2.1:1775764865.815624:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966a00. 00010000:00000010:2.1:1775764865.815635:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916c00. 00000080:00000010:3.1:1775764865.816534:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bc00. 00000080:00000010:3.1:1775764865.820264:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275de300. 00010000:00000010:3.1:1775764865.827034:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789da00. 00000080:00000010:3.1:1775764865.827044:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bc20. 00010000:00000010:3.1:1775764865.827058:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207ce3c00. 00010000:00000010:3.1:1775764865.827063:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2e00. 00000080:00000010:0.1:1775764865.839699:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2e40. 00000080:00000010:0.1:1775764865.839724:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff99422761b9c0. 00010000:00000010:3.1:1775764865.847265:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0200. 00000080:00000010:3.1:1775764865.847289:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8b60. 00010000:00000010:3.1:1775764865.860543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2800. 00010000:00000010:3.1:1775764865.886256:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789da00. 00010000:00000010:2.1:1775764865.896415:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2200. 00000080:00000010:2.1:1775764865.896485:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bc20. 00000080:00000010:2.1:1775764865.896496:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2300. 00010000:00000010:2.1:1775764865.896502:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917e00. 00010000:00000010:3.1:1775764865.901286:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994209db3000. 00010000:00000010:3.1:1775764865.908903:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789da00. 00000080:00000010:3.1:1775764865.917282:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bc00. 00010000:00000010:3.1:1775764865.917293:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916c00. 00010000:00000010:2.1:1775764865.972318:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d22c00. 00010000:00000010:2.1:1775764865.972329:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2800. 00010000:00000010:0.1:1775764865.975461:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994218fe6800. 00010000:00000010:0.1:1775764865.975478:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2200. 00000080:00000010:3.1:1775764865.987391:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db22c0. 00000080:00000010:3.1:1775764865.987430:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227618000. 00000080:00000010:2.1:1775764865.993238:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bdc0. 00000080:00000010:2.1:1775764865.993272:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff99422762c200. 00010000:00000010:3.1:1775764865.999655:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789da00. 00000080:00000010:3.1:1775764865.999674:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bc00. 00010000:00000010:2.1:1775764866.008707:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2800. 00000080:00000010:2.1:1775764866.008730:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2b20. 00010000:00000010:3.1:1775764866.056949:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789da00. 00010000:00000010:2.1:1775764866.079607:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2800. 00010000:00000010:3.1:1775764866.104208:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789da00. 00010000:00000010:3.1:1775764866.104222:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8200. 00010000:00000010:2.1:1775764866.108290:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2200. 00000080:00000010:2.1:1775764866.108340:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2b20. 00000080:00000010:2.1:1775764866.108348:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bc00. 00010000:00000010:2.1:1775764866.108354:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994209db3000. 00010000:00000010:0.1:1775764866.122221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1000. 00000080:00000010:0.1:1775764866.132260:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bd40. 00000080:00000010:0.1:1775764866.132269:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b060. 00010000:00000010:0.1:1775764866.132276:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2a00. 00010000:00000010:3.1:1775764866.136250:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d22c00. 00010000:00000010:3.1:1775764866.136258:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2800. 00000080:00000010:3.1:1775764866.164302:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2040. 00000080:00000010:3.1:1775764866.176218:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227618000. 00010000:00000010:0.1:1775764866.190279:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1c00. 00000080:00000010:0.1:1775764866.190287:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b7c0. 00010000:00000010:3.1:1775764866.197439:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8200. 00010000:00000010:2.1:1775764866.198867:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994218fe6800. 00010000:00000010:2.1:1775764866.198877:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1a00. 00000080:00000010:2.1:1775764866.198882:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c82e0. 00000080:00000010:2.1:1775764866.198898:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff99422762ca40. 00010000:00000010:3.1:1775764866.203203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb9c00. 00010000:00000010:2.1:1775764866.212745:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917e00. 00000080:00000010:2.1:1775764866.212768:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2b20. 00010000:00000010:0.1:1775764866.237240:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1a00. 00000080:00000010:0.1:1775764866.237311:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b800. 00000080:00000010:0.1:1775764866.237320:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6be60. 00010000:00000010:0.1:1775764866.237325:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1600. 00010000:00000010:2.1:1775764866.262318:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917e00. 00010000:00000010:2.1:1775764866.286005:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2800. 00010000:00000010:1.1:1775764866.288706:0:2393:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994220010500. 00010000:00000010:1.1:1775764866.288712:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1c00. 00000080:00000010:3.1:1775764866.297761:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8300. 00000080:00000010:3.1:1775764866.297790:0:32:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff99422762d280. 00000080:00000010:0.1:1775764866.310191:0:2400:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bb20. 00010000:00000010:2.1:1775764866.314254:0:14452:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2a00. 00000080:00000010:2.1:1775764866.314309:0:14452:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2720. 00000080:00000010:2.1:1775764866.314320:0:14452:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2b20. 00010000:00000010:2.1:1775764866.314324:0:14452:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994209db3000. 00010000:00000010:3.1:1775764866.318576:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb9c00. 00010000:00000010:2.1:1775764866.338207:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420387cc00. 00010000:00000010:2.1:1775764866.338215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2800. 00010000:00000010:1.1:1775764866.358352:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a6400. 00000080:00000010:1.1:1775764866.363013:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2700. 00010000:00000010:2.1:1775764866.377191:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a6200. 00000080:00000010:1.1:1775764866.377293:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227607380. 00010000:00000010:2.1:1775764866.389201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1600. 00000080:00000010:2.1:1775764866.389212:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c84a0. 00010000:00000010:0.1:1775764866.389294:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1a00. 00000080:00000010:0.1:1775764866.393965:0:2392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a5a0. 00000080:00000010:0.1:1775764866.393980:0:2392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6aa40. 00010000:00000010:0.1:1775764866.393985:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1c00. 00010000:00000010:3.1:1775764866.437241:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2200. 00010000:00000010:1.1:1775764866.444428:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994220010600. 00010000:00000010:1.1:1775764866.444445:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ce1800. 00010000:00000010:2.1:1775764866.444898:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2a00. 00010000:00000010:2.1:1775764866.450778:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb9c00. 00000080:00000010:2.1:1775764866.450836:0:26:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2720. 00000080:00000010:2.1:1775764866.450843:0:26:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2b20. 00010000:00000010:2.1:1775764866.450847:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2800. 00000080:00000010:1.1:1775764866.461854:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8720. 00000080:00000010:1.1:1775764866.461888:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff99422762d280. 00010000:00000010:1.1:1775764866.476188:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902400. 00000080:00000010:1.1:1775764866.476212:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6afc0. 00010000:00000010:2.1:1775764866.489261:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966f00. 00010000:00000010:2.1:1775764866.489280:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2200. 00010000:00000010:2.1:1775764866.517228:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2a00. 00000080:00000010:3.1:1775764866.518640:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b9c0. 00000080:00000010:3.1:1775764866.518674:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275de300. 00010000:00000010:2.1:1775764866.527355:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2200. 00010000:00000010:3.1:1775764866.550888:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8200. 00000080:00000010:0.1:1775764866.551211:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b160. 00010000:00000010:2.1:1775764866.556010:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1c00. 00000080:00000010:2.1:1775764866.560257:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2720. 00000080:00000010:2.1:1775764866.560272:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2b20. 00010000:00000010:2.1:1775764866.560277:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903c00. 00010000:00000010:1.1:1775764866.569501:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a6200. 00010000:00000010:1.1:1775764866.584283:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a6400. 00010000:00000010:2.1:1775764866.611341:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a6200. 00000080:00000010:3.1:1775764866.611940:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a900. 00000080:00000010:3.1:1775764866.611950:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a820. 00010000:00000010:3.1:1775764866.611955:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6000. 00010000:00000010:3.1:1775764866.619003:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d22f00. 00010000:00000010:3.1:1775764866.619018:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1a00. 00000080:00000010:2.1:1775764866.633494:0:13:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2ee0. 00000080:00000010:2.1:1775764866.654488:0:14454:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff99422761b9c0. 00010000:00000010:2.1:1775764866.659231:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d22100. 00010000:00000010:2.1:1775764866.659253:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8200. 00000080:00000010:2.1:1775764866.664898:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a6a0. 00000080:00000010:2.1:1775764866.664928:0:14454:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227618000. 00010000:00000010:3.1:1775764866.665778:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1a00. 00000080:00000010:3.1:1775764866.665788:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2720. 00010000:00000010:3.1:1775764866.665796:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2200. 00000080:00000010:3.1:1775764866.665799:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2ee0. 00010000:00000010:3.1:1775764866.694543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8200. 00010000:00000010:0.1:1775764866.701404:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1a00. 00010000:00000010:3.1:1775764866.723386:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a6400. 00010000:00000010:3.1:1775764866.723404:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2a00. 00000080:00000010:2.1:1775764866.737568:0:26:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bc80. 00000080:00000010:2.1:1775764866.737588:0:26:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b9c0. 00010000:00000010:2.1:1775764866.737595:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb9c00. 00010000:00000010:2.1:1775764866.757716:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1a00. 00010000:00000010:3.1:1775764866.760218:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2a00. 00010000:00000010:1.1:1775764866.773225:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994220010900. 00010000:00000010:1.1:1775764866.773251:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2200. 00000080:00000010:3.1:1775764866.773563:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bea0. 00000080:00000010:3.1:1775764866.773573:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2100. 00010000:00000010:3.1:1775764866.773579:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8000. 00000080:00000010:3.1:1775764866.787566:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a440. 00000080:00000010:3.1:1775764866.789493:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff99422762ca40. 00010000:00000010:1.1:1775764866.790212:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966e00. 00010000:00000010:1.1:1775764866.790222:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb9c00. 00000080:00000010:1.1:1775764866.799395:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db29a0. 00010000:00000010:3.1:1775764866.800058:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8000. 00000080:00000010:3.1:1775764866.800069:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bea0. 00000080:00000010:1.1:1775764866.808602:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275de300. 00010000:00000010:1.1:1775764866.821072:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903c00. 00000080:00000010:1.1:1775764866.821093:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6aa00. 00010000:00000010:1.1:1775764866.835567:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7400. 00010000:00000010:2.1:1775764866.854224:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2200. 00010000:00000010:1.1:1775764866.862217:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970200. 00010000:00000010:1.1:1775764866.871341:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971a00. 00000080:00000010:0.1:1775764866.871436:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6aba0. 00000080:00000010:0.1:1775764866.871444:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8c40. 00010000:00000010:0.1:1775764866.871449:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8200. 00010000:00000010:2.1:1775764866.872944:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2a00. 00000080:00000010:2.1:1775764866.872989:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db21a0. 00010000:00000010:2.1:1775764866.872996:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902400. 00010000:00000010:0.1:1775764866.899181:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994220010b00. 00010000:00000010:0.1:1775764866.899194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6400. 00010000:00000010:0.1:1775764866.908319:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d22100. 00010000:00000010:0.1:1775764866.908332:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2200. 00000080:00000010:2.1:1775764866.908399:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6abe0. 00000080:00000010:2.1:1775764866.908432:0:14453:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff99422762c200. 00000080:00000010:0.1:1775764866.914221:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2f80. 00000080:00000010:1.1:1775764866.914365:0:2392:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227618000. 00010000:00000010:2.1:1775764866.928129:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1a00. 00000080:00000010:2.1:1775764866.928142:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db21a0. 00010000:00000010:3.1:1775764866.947435:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8000. 00000080:00000010:3.1:1775764866.947451:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bea0. 00010000:00000010:2.1:1775764866.983239:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1a00. 00010000:00000010:2.1:1775764866.996299:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917e00. 00010000:00000010:1.1:1775764867.001206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971a00. 00010000:00000010:1.1:1775764867.001257:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8200. 00010000:00000010:3.1:1775764867.010345:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1a00. 00000080:00000010:3.1:1775764867.010406:0:2402:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2680. 00000080:00000010:3.1:1775764867.010426:0:2402:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db21a0. 00010000:00000010:3.1:1775764867.010430:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994209db3000. 00010000:00000010:0.1:1775764867.070184:0:14461:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207c60200. 00010000:00000010:0.1:1775764867.070191:0:14461:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789c800. 00010000:00000010:0.1:1775764867.070196:0:14461:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420b307300. 00010000:00000010:0.1:1775764867.070199:0:14461:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da000. 00010000:00000010:1.1:1775764867.072434:0:2392:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d22c00. 00010000:00000010:1.1:1775764867.072443:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970200. 00010000:00000010:3.1:1775764867.096176:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942420d9d00. 00010000:00000010:3.1:1775764867.096187:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420b14a400. 00010000:00000010:3.1:1775764867.096193:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994218fe6400. 00010000:00000010:3.1:1775764867.096197:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994201483e00. 00010000:00000010:3.1:1775764867.096203:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078a1a00. 00010000:00000010:3.1:1775764867.096207:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970000. 00010000:00000010:3.1:1775764867.096212:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078e9000. 00010000:00000010:3.1:1775764867.096217:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca0c00. 00010000:00000010:3.1:1775764867.096221:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207c60b00. 00010000:00000010:3.1:1775764867.096224:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca0000. 00010000:00000010:1.1:1775764867.096391:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994211bdb300. 00010000:00000010:1.1:1775764867.096398:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc600. 00010000:00000010:1.1:1775764867.096403:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078b3f00. 00010000:00000010:1.1:1775764867.096406:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916a00. 00010000:00000010:1.1:1775764867.096410:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209d57100. 00010000:00000010:1.1:1775764867.096425:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894400. 00010000:00000010:1.1:1775764867.096428:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078e9300. 00010000:00000010:1.1:1775764867.096432:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1600. 00010000:00000010:1.1:1775764867.096436:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420788e600. 00010000:00000010:1.1:1775764867.096440:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994209db2a00. 00010000:00000010:1.1:1775764867.096456:0:20:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078b3800. 00010000:00000010:1.1:1775764867.096460:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793d200. 00010000:00000010:1.1:1775764867.096486:0:20:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994211bdb900. 00010000:00000010:1.1:1775764867.096489:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793c000. 00010000:00000010:2.1:1775764867.096806:0:2403:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207c70600. 00010000:00000010:2.1:1775764867.096813:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917a00. 00010000:00000010:1.1:1775764867.102994:0:2392:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207eef500. 00010000:00000010:1.1:1775764867.103002:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075cac00. 00010000:00000010:1.1:1775764867.103006:0:2392:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943b00. 00010000:00000010:1.1:1775764867.103009:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895800. 00010000:00000010:1.1:1775764867.103012:0:2392:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207eef900. 00010000:00000010:1.1:1775764867.103015:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca0e00. 00010000:00000010:1.1:1775764867.103018:0:2392:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078b3200. 00010000:00000010:1.1:1775764867.103021:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894600. 00010000:00000010:3.1:1775764867.141711:0:32:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207ce3600. 00010000:00000010:3.1:1775764867.141721:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916000. 00010000:00000010:2.1:1775764867.147693:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207931000. 00010000:00000010:2.1:1775764867.147703:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895400. 00010000:00000010:0.1:1775764867.147759:0:2393:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420387c500. 00010000:00000010:0.1:1775764867.147766:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917800. 00000080:00000010:3.1:1775764867.147815:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6aa40. 00000080:00000010:3.1:1775764867.147838:0:32:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff99422761b9c0. 00010000:00000010:2.1:1775764867.163241:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943500. 00010000:00000010:2.1:1775764867.163248:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dba00. 00010000:00000010:2.1:1775764867.163252:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943000. 00010000:00000010:2.1:1775764867.163255:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207938600. 00010000:00000010:2.1:1775764867.163260:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420387c100. 00010000:00000010:2.1:1775764867.163263:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917c00. 00010000:00000010:2.1:1775764867.163268:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207c67600. 00010000:00000010:2.1:1775764867.163271:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0a00. 00010000:00000010:2.1:1775764867.163275:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99421263ae00. 00010000:00000010:2.1:1775764867.163278:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8000. 00010000:00000010:1.1:1775764867.168195:0:2401:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207eeff00. 00010000:00000010:1.1:1775764867.168221:0:2401:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075cbc00. 00010000:00000010:1.1:1775764867.168228:0:2401:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966b00. 00010000:00000010:1.1:1775764867.168231:0:2401:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894c00. 00010000:00000010:2.1:1775764867.170296:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966700. 00010000:00000010:2.1:1775764867.170319:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075dbe00. 00010000:00000010:2.1:1775764867.170326:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994211cc1700. 00010000:00000010:2.1:1775764867.170330:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1800. 00010000:00000010:1.1:1775764867.179231:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994210812000. 00010000:00000010:1.1:1775764867.179248:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb3800. 00010000:00000010:1.1:1775764867.179254:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d22b00. 00010000:00000010:1.1:1775764867.179258:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916c00. 00010000:00000010:1.1:1775764867.179261:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d22700. 00010000:00000010:1.1:1775764867.179264:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2e00. 00010000:00000010:3.1:1775764867.179455:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994220010c00. 00010000:00000010:3.1:1775764867.179461:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1000. 00010000:00000010:2.1:1775764867.179541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb9c00. 00000080:00000010:2.1:1775764867.179549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bea0. 00010000:00000010:2.1:1775764867.179560:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420387cd00. 00010000:00000010:2.1:1775764867.179563:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a6a00. 00010000:00000010:2.1:1775764867.179566:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d22400. 00010000:00000010:2.1:1775764867.179568:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2800. 00000080:00000010:2.1:1775764867.184223:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2840. 00000080:00000010:2.1:1775764867.184260:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227618000. 00010000:00000010:3.1:1775764867.219239:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075cbc00. 00000080:00000010:1.1:1775764867.223286:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2e60. 00010000:00000010:2.1:1775764867.237475:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a6a00. 00010000:00000010:2.1:1775764867.237529:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075cac00. 00010000:00000010:2.1:1775764867.264198:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207938600. 00010000:00000010:3.1:1775764867.273611:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7e00. 00010000:00000010:2.1:1775764867.282588:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a6a00. 00000080:00000010:2.1:1775764867.282640:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2840. 00010000:00000010:2.1:1775764867.282647:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971a00. 00010000:00000010:1.1:1775764867.334208:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966800. 00010000:00000010:1.1:1775764867.334219:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793d200. 00000080:00000010:3.1:1775764867.344066:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2660. 00000080:00000010:3.1:1775764867.344104:0:14454:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275ddac0. 00010000:00000010:3.1:1775764867.368373:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8000. 00000080:00000010:3.1:1775764867.368386:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bbe0. 00010000:00000010:3.1:1775764867.382591:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075cac00. 00000080:00000010:3.1:1775764867.382608:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b580. 00010000:00000010:3.1:1775764867.403876:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8000. 00010000:00000010:2.1:1775764867.410667:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8200. 00010000:00000010:0.1:1775764867.414756:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7000. 00010000:00000010:0.1:1775764867.425656:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6c00. 00010000:00000010:2.1:1775764867.437181:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994206989500. 00010000:00000010:2.1:1775764867.437191:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793d800. 00010000:00000010:1.1:1775764867.441193:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7000. 00000080:00000010:2.1:1775764867.446560:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bbe0. 00000080:00000010:2.1:1775764867.446606:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227618000. 00010000:00000010:1.1:1775764867.450197:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971000. 00010000:00000010:2.1:1775764867.462909:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970000. 00000080:00000010:3.1:1775764867.469251:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2b20. 00010000:00000010:2.1:1775764867.473233:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793d800. 00010000:00000010:2.1:1775764867.473240:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970e00. 00010000:00000010:3.1:1775764867.500015:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078efd00. 00010000:00000010:3.1:1775764867.500034:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c600. 00010000:00000010:1.1:1775764867.521500:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6600. 00010000:00000010:1.1:1775764867.521557:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8000. 00010000:00000010:3.1:1775764867.530220:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6e00. 00000080:00000010:3.1:1775764867.558446:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a420. 00010000:00000010:2.1:1775764867.562085:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793d800. 00010000:00000010:3.1:1775764867.594977:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8000. 00010000:00000010:2.1:1775764867.621576:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966c00. 00010000:00000010:2.1:1775764867.621588:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8200. 00010000:00000010:3.1:1775764867.638222:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793d800. 00010000:00000010:2.1:1775764867.663050:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207938600. 00000080:00000010:2.1:1775764867.663073:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2520. 00000080:00000010:1.1:1775764867.663379:0:13:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b580. 00010000:00000010:3.1:1775764867.676287:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971a00. 00010000:00000010:3.1:1775764867.680370:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994220010000. 00010000:00000010:3.1:1775764867.680378:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971800. 00010000:00000010:2.1:1775764867.715235:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970200. 00010000:00000010:2.1:1775764867.742318:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970e00. 00000080:00000010:0.1:1775764867.748312:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a820. 00000080:00000010:0.1:1775764867.748323:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8720. 00000080:00000010:0.1:1775764867.748328:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8300. 00000080:00000010:0.1:1775764867.748332:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c81e0. 00010000:00000010:0.1:1775764867.748339:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075cbc00. 00010000:00000010:3.1:1775764867.762923:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8000. 00010000:00000010:3.1:1775764867.776885:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8200. 00010000:00000010:0.1:1775764867.803269:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81600. 00010000:00000010:0.1:1775764867.817263:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6e00. 00000080:00000010:2.1:1775764867.817937:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6abe0. 00010000:00000010:1.1:1775764867.822065:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994220010300. 00010000:00000010:1.1:1775764867.822074:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6200. 00000080:00000010:2.1:1775764867.822378:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6aac0. 00000080:00000010:2.1:1775764867.822401:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff99422762dac0. 00010000:00000010:3.1:1775764867.822647:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6800. 00010000:00000010:3.1:1775764867.830457:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6000. 00010000:00000010:2.1:1775764867.850745:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80e00. 00000080:00000010:2.1:1775764867.850755:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2540. 00010000:00000010:0.1:1775764867.872333:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d22e00. 00010000:00000010:0.1:1775764867.872341:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd000. 00010000:00000010:2.1:1775764867.902237:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80a00. 00010000:00000010:3.1:1775764867.911268:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80000. 00010000:00000010:2.1:1775764867.922877:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80c00. 00000080:00000010:2.1:1775764867.922951:0:26:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2300. 00010000:00000010:2.1:1775764867.922959:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81200. 00000080:00000010:2.1:1775764867.933924:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8300. 00010000:00000010:3.1:1775764867.940459:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81800. 00010000:00000010:1.1:1775764867.951234:0:2392:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207931000. 00010000:00000010:1.1:1775764867.951257:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80e00. 00010000:00000010:2.1:1775764867.959232:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420387c100. 00010000:00000010:2.1:1775764867.959248:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80a00. 00000080:00000010:2.1:1775764867.967252:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2bc0. 00000080:00000010:2.1:1775764867.967275:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227607380. 00010000:00000010:3.1:1775764867.970144:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81200. 00010000:00000010:2.1:1775764867.977312:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80c00. 00000080:00000010:2.1:1775764867.977326:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2300. 00000080:00000010:1.1:1775764867.982548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2540. 00010000:00000010:3.1:1775764867.999532:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970e00. 00010000:00000010:2.1:1775764868.001183:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6000. 00010000:00000010:3.1:1775764868.009245:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966f00. 00010000:00000010:3.1:1775764868.009253:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8000. 00010000:00000010:3.1:1775764868.013202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81200. 00000080:00000010:3.1:1775764868.013264:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8980. 00010000:00000010:3.1:1775764868.013286:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80a00. 00010000:00000010:0.1:1775764868.023179:0:2401:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80c00. 00010000:00000010:1.1:1775764868.039440:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420387c900. 00010000:00000010:1.1:1775764868.039449:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793d800. 00010000:00000010:2.1:1775764868.042937:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81800. 00000080:00000010:2.1:1775764868.042991:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8720. 00000080:00000010:2.1:1775764868.043001:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2520. 00000080:00000010:2.1:1775764868.043007:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b140. 00000080:00000010:2.1:1775764868.043012:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b7e0. 00010000:00000010:2.1:1775764868.043015:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80400. 00000080:00000010:1.1:1775764868.059203:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2300. 00000080:00000010:1.1:1775764868.059222:0:14454:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227600000. 00010000:00000010:1.1:1775764868.070072:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971800. 00000080:00000010:1.1:1775764868.070082:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a4c0. 00000080:00000010:2.1:1775764868.090888:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a760. 00010000:00000010:0.1:1775764868.092246:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970e00. 00010000:00000010:2.1:1775764868.093214:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207976700. 00010000:00000010:2.1:1775764868.093223:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80000. 00000080:00000010:2.1:1775764868.102454:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db27a0. 00000080:00000010:2.1:1775764868.102483:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff99422761b9c0. 00010000:00000010:2.1:1775764868.115755:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80000. 00000080:00000010:2.1:1775764868.115774:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2bc0. 00010000:00000010:0.1:1775764868.133241:0:2392:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420387c900. 00010000:00000010:0.1:1775764868.133253:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6c00. 00000080:00000010:0.1:1775764868.165036:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b840. 00000080:00000010:0.1:1775764868.165061:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227600000. 00010000:00000010:2.1:1775764868.165495:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80000. 00010000:00000010:2.1:1775764868.165513:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81600. 00000080:00000010:2.1:1775764868.182505:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2bc0. 00010000:00000010:2.1:1775764868.182525:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81800. 00010000:00000010:2.1:1775764868.182530:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970200. 00000080:00000010:2.1:1775764868.182535:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6aba0. 00010000:00000010:3.1:1775764868.210342:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81600. 00010000:00000010:2.1:1775764868.218717:0:2402:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99421835ab00. 00010000:00000010:2.1:1775764868.218725:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80c00. 00000080:00000010:1.1:1775764868.225214:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db27a0. 00000080:00000010:1.1:1775764868.237238:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff99422761b9c0. 00010000:00000010:2.1:1775764868.241202:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966f00. 00010000:00000010:2.1:1775764868.241209:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793d800. 00010000:00000010:1.1:1775764868.249209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894400. 00000080:00000010:1.1:1775764868.249223:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a6a0. 00000080:00000010:2.1:1775764868.257197:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b180. 00000080:00000010:2.1:1775764868.257225:0:14454:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275ddac0. 00010000:00000010:2.1:1775764868.271220:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80c00. 00000080:00000010:2.1:1775764868.271235:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2bc0. 00010000:00000010:0.1:1775764868.292983:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6600. 00010000:00000010:0.1:1775764868.299711:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6000. 00000080:00000010:0.1:1775764868.304459:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8300. 00010000:00000010:0.1:1775764868.304474:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894600. 00010000:00000010:2.1:1775764868.311635:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81800. 00010000:00000010:1.1:1775764868.335286:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420387cc00. 00010000:00000010:1.1:1775764868.335296:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6600. 00010000:00000010:2.1:1775764868.338719:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99421835ab00. 00010000:00000010:2.1:1775764868.338730:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80c00. 00000080:00000010:1.1:1775764868.346497:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8320. 00000080:00000010:1.1:1775764868.346530:0:14453:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227603180. 00000080:00000010:2.1:1775764868.353208:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2bc0. 00000080:00000010:2.1:1775764868.353232:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff99422761b9c0. 00010000:00000010:1.1:1775764868.357863:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894c00. 00000080:00000010:1.1:1775764868.357874:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a760. 00010000:00000010:2.1:1775764868.361191:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80c00. 00000080:00000010:2.1:1775764868.361198:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2a20. 00010000:00000010:2.1:1775764868.388509:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80a00. 00010000:00000010:3.1:1775764868.389210:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895400. 00010000:00000010:2.1:1775764868.418267:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966f00. 00010000:00000010:2.1:1775764868.418292:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971800. 00010000:00000010:2.1:1775764868.426492:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99421835ab00. 00010000:00000010:2.1:1775764868.426503:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80c00. 00000080:00000010:3.1:1775764868.435371:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2a20. 00000080:00000010:2.1:1775764868.440357:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b4c0. 00000080:00000010:2.1:1775764868.440377:0:14454:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275dd280. 00000080:00000010:3.1:1775764868.441449:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff99422761b9c0. 00010000:00000010:3.1:1775764868.448647:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075cbc00. 00010000:00000010:1.1:1775764868.451183:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80c00. 00000080:00000010:1.1:1775764868.451195:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2420. 00000080:00000010:2.1:1775764868.454231:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b440. 00010000:00000010:2.1:1775764868.487919:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894600. 00010000:00000010:2.1:1775764868.496212:0:1005:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81200. 00010000:00000010:1.1:1775764868.543920:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894600. 00000080:00000010:1.1:1775764868.544005:0:20:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2640. 00010000:00000010:1.1:1775764868.544016:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7a00. 00010000:00000010:3.1:1775764868.546634:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420387cc00. 00010000:00000010:3.1:1775764868.547392:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80c00. 00000080:00000010:3.1:1775764868.547415:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2820. 00000080:00000010:3.1:1775764868.547440:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227603180. 00010000:00000010:3.1:1775764868.562483:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075cbc00. 00000080:00000010:3.1:1775764868.578017:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b480. 00010000:00000010:0.1:1775764868.599389:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420387c900. 00010000:00000010:0.1:1775764868.599429:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895400. 00000080:00000010:3.1:1775764868.623518:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2be0. 00010000:00000010:2.1:1775764868.624245:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075cac00. 00000080:00000010:3.1:1775764868.633288:0:14454:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942276018c0. 00010000:00000010:3.1:1775764868.652030:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075cbc00. 00000080:00000010:2.1:1775764868.657201:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b480. 00010000:00000010:2.1:1775764868.676374:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99421835ab00. 00010000:00000010:2.1:1775764868.676407:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81200. 00010000:00000010:3.1:1775764868.704263:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81200. 00000080:00000010:2.1:1775764868.708478:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2480. 00000080:00000010:2.1:1775764868.708518:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff99422761b9c0. 00010000:00000010:2.1:1775764868.725955:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971800. 00000080:00000010:2.1:1775764868.725967:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2d80. 00010000:00000010:3.1:1775764868.742038:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075cbc00. 00010000:00000010:0.1:1775764868.756283:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895400. 00000080:00000010:1.1:1775764868.763298:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b0a0. 00000080:00000010:1.1:1775764868.763309:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db25e0. 00010000:00000010:1.1:1775764868.763316:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80c00. 00010000:00000010:3.1:1775764868.771473:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793d800. 00010000:00000010:2.1:1775764868.792794:0:13471:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966f00. 00010000:00000010:2.1:1775764868.792806:0:13471:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075cbc00. 00010000:00000010:0.1:1775764868.817217:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420387c100. 00010000:00000010:0.1:1775764868.817231:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894c00. 00010000:00000010:0.1:1775764868.822537:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075cbc00. 00000080:00000010:0.1:1775764868.822618:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b8a0. 00000080:00000010:3.1:1775764868.823634:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b8c0. 00000080:00000010:3.1:1775764868.823660:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275da940. 00000080:00000010:3.1:1775764868.823676:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a340. 00000080:00000010:3.1:1775764868.823688:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227604200. 00010000:00000010:3.1:1775764868.828224:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80c00. 00000080:00000010:3.1:1775764868.828241:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b080. 00010000:00000010:2.1:1775764868.864469:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793d800. 00010000:00000010:0.1:1775764868.864620:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971800. 00010000:00000010:2.1:1775764868.866439:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207938600. 00000080:00000010:2.1:1775764868.866485:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2480. 00010000:00000010:2.1:1775764868.866493:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6e00. 00010000:00000010:3.1:1775764868.868615:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895400. 00010000:00000010:2.1:1775764868.878542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894c00. 00000080:00000010:2.1:1775764868.890247:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a6a0. 00000080:00000010:2.1:1775764868.890256:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db22a0. 00010000:00000010:2.1:1775764868.890262:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81200. 00010000:00000010:0.1:1775764868.896281:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99421835ab00. 00010000:00000010:0.1:1775764868.896288:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994209db2a00. 00000080:00000010:0.1:1775764868.901207:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2fa0. 00000080:00000010:0.1:1775764868.901239:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff99422761b9c0. 00010000:00000010:0.1:1775764868.910490:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6000. 00010000:00000010:2.1:1775764868.913328:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99421835ae00. 00010000:00000010:2.1:1775764868.913337:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207938600. 00000080:00000010:0.1:1775764868.917327:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8920. 00000080:00000010:2.1:1775764868.929034:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a340. 00000080:00000010:2.1:1775764868.929068:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275b8000. 00010000:00000010:2.1:1775764868.943711:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207938600. 00000080:00000010:2.1:1775764868.943735:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db22a0. 00010000:00000010:3.1:1775764868.960096:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895400. 00010000:00000010:3.1:1775764868.975741:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ce0800. 00010000:00000010:2.1:1775764869.000993:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7e00. 00000080:00000010:2.1:1775764869.001068:0:26:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b080. 00000080:00000010:2.1:1775764869.001078:0:26:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8140. 00010000:00000010:2.1:1775764869.001084:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6200. 00010000:00000010:3.1:1775764869.001343:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207938600. 00010000:00000010:1.1:1775764869.015288:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80c00. 00000080:00000010:2.1:1775764869.027230:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b8c0. 00000080:00000010:2.1:1775764869.027239:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db22a0. 00010000:00000010:2.1:1775764869.027245:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ce1800. 00010000:00000010:3.1:1775764869.027823:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895400. 00010000:00000010:0.1:1775764869.028957:0:2393:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99421835ae00. 00010000:00000010:0.1:1775764869.028967:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1000. 00000080:00000010:1.1:1775764869.042343:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8000. 00000080:00000010:1.1:1775764869.042376:0:14453:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275b8000. 00010000:00000010:1.1:1775764869.058492:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894c00. 00000080:00000010:1.1:1775764869.058501:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a280. 00010000:00000010:3.1:1775764869.066311:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207976700. 00010000:00000010:3.1:1775764869.066328:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0a00. 00000080:00000010:3.1:1775764869.077944:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6aac0. 00000080:00000010:3.1:1775764869.077987:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275bb9c0. 00010000:00000010:2.1:1775764869.081405:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1000. 00010000:00000010:3.1:1775764869.091860:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80c00. 00000080:00000010:3.1:1775764869.091876:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b5c0. 00010000:00000010:0.1:1775764869.093266:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0a00. 00010000:00000010:2.1:1775764869.104480:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1000. 00000080:00000010:2.1:1775764869.104547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db28a0. 00000080:00000010:2.1:1775764869.116495:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2120. 00010000:00000010:2.1:1775764869.116507:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895400. 00010000:00000010:3.1:1775764869.130203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789c800. 00010000:00000010:3.1:1775764869.130219:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80c00. 00010000:00000010:3.1:1775764869.149758:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895400. 00000080:00000010:2.1:1775764869.154241:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b760. 00000080:00000010:2.1:1775764869.154250:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b5c0. 00010000:00000010:2.1:1775764869.154254:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81200. 00010000:00000010:0.1:1775764869.158213:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207de4e00. 00010000:00000010:0.1:1775764869.158230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0a00. 00000080:00000010:1.1:1775764869.158293:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2780. 00000080:00000010:1.1:1775764869.162416:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275bb180. 00010000:00000010:1.1:1775764869.173495:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916000. 00000080:00000010:1.1:1775764869.177388:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a760. 00010000:00000010:1.1:1775764869.205231:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207de4f00. 00010000:00000010:1.1:1775764869.205266:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789c800. 00000080:00000010:2.1:1775764869.214688:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b660. 00000080:00000010:2.1:1775764869.223566:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275b98c0. 00010000:00000010:1.1:1775764869.235245:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916000. 00010000:00000010:2.1:1775764869.242503:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0a00. 00000080:00000010:2.1:1775764869.242514:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2900. 00010000:00000010:1.1:1775764869.245380:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7e00. 00010000:00000010:2.1:1775764869.272308:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916000. 00000080:00000010:2.1:1775764869.290654:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8980. 00000080:00000010:2.1:1775764869.290666:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a760. 00010000:00000010:2.1:1775764869.290674:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917800. 00010000:00000010:1.1:1775764869.389236:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00010000:00000010:2.1:1775764869.414645:0:36:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207de4100. 00010000:00000010:2.1:1775764869.414658:0:36:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898400. 00010000:00000010:1.1:1775764869.436378:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917a00. 00010000:00000010:0.1:1775764869.447031:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789c800. 00000080:00000010:0.1:1775764869.447192:0:2400:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a440. 00000080:00000010:0.1:1775764869.447200:0:2400:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db21e0. 00010000:00000010:0.1:1775764869.447206:0:2400:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899200. 00000080:00000010:3.1:1775764869.452591:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a900. 00000080:00000010:3.1:1775764869.452613:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275b98c0. 00010000:00000010:2.1:1775764869.455433:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895400. 00000080:00000010:2.1:1775764869.455454:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b520. 00010000:00000010:0.1:1775764869.528905:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994220010500. 00010000:00000010:0.1:1775764869.528914:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7c00. 00010000:00000010:2.1:1775764869.530186:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0800. 00000080:00000010:0.1:1775764869.530195:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a660. 00000080:00000010:0.1:1775764869.530213:0:14454:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227628840. 00010000:00000010:2.1:1775764869.542039:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899200. 00010000:00000010:2.1:1775764869.542070:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6000. 00000080:00000010:2.1:1775764869.542077:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8b60. 00010000:00000010:2.1:1775764869.546737:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00000080:00000010:2.1:1775764869.546788:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2ea0. 00000080:00000010:2.1:1775764869.546794:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a780. 00010000:00000010:2.1:1775764869.546800:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898400. 00010000:00000010:3.1:1775764869.569136:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207de4300. 00010000:00000010:3.1:1775764869.569178:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207535c00. 00010000:00000010:2.1:1775764869.573185:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903c00. 00000080:00000010:3.1:1775764869.579053:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db25c0. 00000080:00000010:3.1:1775764869.579085:0:32:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275b9080. 00010000:00000010:2.1:1775764869.581345:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898400. 00000080:00000010:2.1:1775764869.581413:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2ea0. 00010000:00000010:2.1:1775764869.581422:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0a00. 00010000:00000010:0.1:1775764869.616437:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7c00. 00000080:00000010:1.1:1775764869.631212:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8140. 00010000:00000010:2.1:1775764869.673071:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207de4700. 00010000:00000010:2.1:1775764869.673081:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00010000:00000010:0.1:1775764869.678252:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7c00. 00000080:00000010:2.1:1775764869.678254:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2360. 00000080:00000010:2.1:1775764869.678288:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275bc200. 00010000:00000010:2.1:1775764869.678301:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899200. 00000080:00000010:2.1:1775764869.678305:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2d60. 00010000:00000010:0.1:1775764869.699341:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7000. 00010000:00000010:3.1:1775764869.725681:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899200. 00010000:00000010:1.1:1775764869.732311:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0800. 00000080:00000010:1.1:1775764869.732364:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8660. 00000080:00000010:1.1:1775764869.732372:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c83c0. 00010000:00000010:1.1:1775764869.732376:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1c00. 00010000:00000010:0.1:1775764869.742214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894400. 00010000:00000010:2.1:1775764869.763244:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1c00. 00000080:00000010:2.1:1775764869.779445:0:2402:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bee0. 00000080:00000010:2.1:1775764869.779456:0:2402:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2d60. 00010000:00000010:2.1:1775764869.779463:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00010000:00000010:0.1:1775764869.787349:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207de4700. 00010000:00000010:0.1:1775764869.787373:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1800. 00000080:00000010:1.1:1775764869.793892:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6aba0. 00000080:00000010:1.1:1775764869.793914:0:14454:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275bc200. 00010000:00000010:2.1:1775764869.811897:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0600. 00000080:00000010:2.1:1775764869.811919:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a980. 00010000:00000010:0.1:1775764869.829200:0:2393:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994218fe6400. 00010000:00000010:0.1:1775764869.829230:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0a00. 00000080:00000010:3.1:1775764869.829450:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a880. 00000080:00000010:3.1:1775764869.839409:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275da940. 00010000:00000010:3.1:1775764869.849214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894c00. 00000080:00000010:3.1:1775764869.849224:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b920. 00010000:00000010:2.1:1775764869.875308:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1600. 00010000:00000010:3.1:1775764869.891914:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894c00. 00010000:00000010:3.1:1775764869.891922:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0e00. 00000080:00000010:2.1:1775764869.904433:0:26:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2d60. 00010000:00000010:2.1:1775764869.904443:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00010000:00000010:2.1:1775764869.921892:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0600. 00010000:00000010:2.1:1775764869.942314:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0e00. 00000080:00000010:2.1:1775764869.942363:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2f00. 00000080:00000010:2.1:1775764869.942372:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b920. 00010000:00000010:2.1:1775764869.942378:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894400. 00010000:00000010:3.1:1775764869.942448:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994218fe6400. 00010000:00000010:3.1:1775764869.942455:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1600. 00010000:00000010:2.1:1775764869.960220:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d0bb00. 00010000:00000010:2.1:1775764869.960237:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00000080:00000010:3.1:1775764869.964195:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2140. 00000080:00000010:3.1:1775764869.964219:0:14453:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275da940. 00000080:00000010:2.1:1775764869.971289:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db25c0. 00000080:00000010:2.1:1775764869.971316:0:14453:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275bdac0. 00010000:00000010:2.1:1775764869.975203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7000. 00000080:00000010:2.1:1775764869.975224:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2f00. 00010000:00000010:3.1:1775764869.978559:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894c00. 00000080:00000010:3.1:1775764869.978572:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bf40. 00010000:00000010:2.1:1775764870.030496:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7000. 00010000:00000010:2.1:1775764870.030526:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00010000:00000010:2.1:1775764870.040352:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1600. 00010000:00000010:2.1:1775764870.051350:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0e00. 00000080:00000010:2.1:1775764870.051404:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db25c0. 00010000:00000010:2.1:1775764870.051412:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6600. 00010000:00000010:3.1:1775764870.052502:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0600. 00000080:00000010:3.1:1775764870.052559:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db21e0. 00000080:00000010:3.1:1775764870.052568:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2f00. 00010000:00000010:3.1:1775764870.052576:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894400. 00010000:00000010:2.1:1775764870.094214:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d0bb00. 00010000:00000010:2.1:1775764870.094240:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1600. 00010000:00000010:2.1:1775764870.099456:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207c70e00. 00010000:00000010:2.1:1775764870.099472:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894c00. 00000080:00000010:2.1:1775764870.104196:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2fa0. 00000080:00000010:2.1:1775764870.104219:0:14454:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275da940. 00000080:00000010:2.1:1775764870.119328:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2780. 00000080:00000010:2.1:1775764870.123267:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275bdac0. 00010000:00000010:3.1:1775764870.138479:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1600. 00000080:00000010:3.1:1775764870.138490:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db25e0. 00010000:00000010:3.1:1775764870.151138:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00000080:00000010:3.1:1775764870.151309:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2fa0. 00010000:00000010:2.1:1775764870.172218:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6600. 00010000:00000010:2.1:1775764870.200444:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894c00. 00010000:00000010:3.1:1775764870.202139:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7000. 00010000:00000010:2.1:1775764870.232696:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1400. 00000080:00000010:2.1:1775764870.233000:0:26:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2be0. 00000080:00000010:2.1:1775764870.233009:0:26:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2780. 00010000:00000010:2.1:1775764870.233014:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894400. 00010000:00000010:1.1:1775764870.259976:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d0bb00. 00010000:00000010:1.1:1775764870.259995:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6600. 00010000:00000010:2.1:1775764870.268358:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420387c100. 00010000:00000010:2.1:1775764870.268367:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c000. 00000080:00000010:1.1:1775764870.281107:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a760. 00000080:00000010:1.1:1775764870.281132:0:14454:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227604a40. 00010000:00000010:2.1:1775764870.291429:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7400. 00000080:00000010:2.1:1775764870.291440:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8ec0. 00000080:00000010:1.1:1775764870.292227:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2820. 00000080:00000010:1.1:1775764870.292251:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275bdac0. 00010000:00000010:2.1:1775764870.350210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7c00. 00000080:00000010:2.1:1775764870.350220:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8660. 00010000:00000010:2.1:1775764870.360456:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d600. 00010000:00000010:3.1:1775764870.361418:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c800. 00000080:00000010:3.1:1775764870.383328:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2780. 00010000:00000010:3.1:1775764870.383339:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c000. 00010000:00000010:3.1:1775764870.427107:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0200. 00010000:00000010:1.1:1775764870.427290:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943000. 00010000:00000010:1.1:1775764870.427296:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894400. 00000080:00000010:2.1:1775764870.433299:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2be0. 00000080:00000010:2.1:1775764870.433332:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227604a40. 00010000:00000010:3.1:1775764870.437413:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6800. 00010000:00000010:2.1:1775764870.438200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894c00. 00000080:00000010:2.1:1775764870.438209:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bf40. 00010000:00000010:3.1:1775764870.458201:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894400. 00010000:00000010:2.1:1775764870.489308:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d600. 00010000:00000010:2.1:1775764870.489322:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c000. 00010000:00000010:2.1:1775764870.500229:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd000. 00010000:00000010:2.1:1775764870.513736:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc600. 00010000:00000010:0.1:1775764870.514656:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943000. 00010000:00000010:0.1:1775764870.514664:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c800. 00010000:00000010:2.1:1775764870.527208:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd000. 00000080:00000010:0.1:1775764870.531224:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2a20. 00000080:00000010:0.1:1775764870.531265:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227600840. 00010000:00000010:2.1:1775764870.543280:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc600. 00010000:00000010:3.1:1775764870.544718:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1800. 00000080:00000010:3.1:1775764870.544742:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a260. 00010000:00000010:0.1:1775764870.588660:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd000. 00010000:00000010:1.1:1775764870.596694:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0600. 00010000:00000010:2.1:1775764870.606200:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc600. 00010000:00000010:2.1:1775764870.654254:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c000. 00010000:00000010:3.1:1775764870.654310:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd000. 00010000:00000010:3.1:1775764870.654322:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207976e00. 00010000:00000010:3.1:1775764870.654327:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899400. 00010000:00000010:3.1:1775764870.663222:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d600. 00010000:00000010:3.1:1775764870.663249:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894c00. 00010000:00000010:2.1:1775764870.682210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc600. 00010000:00000010:2.1:1775764870.696819:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd000. 00010000:00000010:2.1:1775764870.696840:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c800. 00010000:00000010:2.1:1775764870.696843:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d600. 00010000:00000010:0.1:1775764870.714681:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c000. 00010000:00000010:3.1:1775764870.720270:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0c00. 00000080:00000010:3.1:1775764870.725200:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2660. 00010000:00000010:2.1:1775764870.750274:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894c00. 00010000:00000010:2.1:1775764870.750281:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943300. 00010000:00000010:2.1:1775764870.750284:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898e00. 00010000:00000010:2.1:1775764870.754219:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207c70e00. 00010000:00000010:2.1:1775764870.754238:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1e00. 00010000:00000010:3.1:1775764870.778229:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d600. 00010000:00000010:1.1:1775764870.781959:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6600. 00000080:00000010:2.1:1775764870.816682:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b100. 00000080:00000010:3.1:1775764870.823493:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2ee0. 00010000:00000010:0.1:1775764870.840968:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d600. 00010000:00000010:2.1:1775764870.863801:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6600. 00010000:00000010:1.1:1775764870.865320:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d0be00. 00010000:00000010:1.1:1775764870.865330:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c800. 00010000:00000010:0.1:1775764870.875221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c000. 00010000:00000010:2.1:1775764870.875413:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207c70e00. 00010000:00000010:2.1:1775764870.875421:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7000. 00010000:00000010:3.1:1775764870.898242:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898e00. 00000080:00000010:2.1:1775764870.906127:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2720. 00010000:00000010:1.1:1775764870.929366:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c000. 00000080:00000010:2.1:1775764870.943012:0:6168:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bf40. 00010000:00000010:2.1:1775764870.943038:0:6168:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207976e00. 00010000:00000010:2.1:1775764870.943044:0:6168:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899400. 00010000:00000010:0.1:1775764870.946200:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c800. 00010000:00000010:2.1:1775764870.958546:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994218fe6400. 00010000:00000010:2.1:1775764870.958558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00010000:00000010:2.1:1775764870.965405:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7400. 00010000:00000010:2.1:1775764870.984286:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898e00. 00000080:00000010:1.1:1775764871.005291:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2720. 00010000:00000010:2.1:1775764871.015199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d600. 00010000:00000010:0.1:1775764871.019680:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0400. 00010000:00000010:3.1:1775764871.052288:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898e00. 00010000:00000010:2.1:1775764871.068417:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994220010300. 00010000:00000010:2.1:1775764871.068425:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6400. 00010000:00000010:3.1:1775764871.077228:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00010000:00000010:2.1:1775764871.077300:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1e00. 00010000:00000010:3.1:1775764871.087070:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d600. 00010000:00000010:2.1:1775764871.096208:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898e00. 00010000:00000010:2.1:1775764871.096246:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c800. 00010000:00000010:3.1:1775764871.119945:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c000. 00010000:00000010:1.1:1775764871.137422:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7400. 00010000:00000010:3.1:1775764871.138429:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7000. 00010000:00000010:3.1:1775764871.154245:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00010000:00000010:1.1:1775764871.154653:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0800. 00010000:00000010:1.1:1775764871.154672:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1c00. 00010000:00000010:0.1:1775764871.173463:0:6159:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c800. 00010000:00000010:2.1:1775764871.180010:0:6168:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c000. 00010000:00000010:3.1:1775764871.193556:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d600. 00010000:00000010:2.1:1775764871.205199:0:2403:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207ce3700. 00010000:00000010:2.1:1775764871.205207:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb3000. 00010000:00000010:3.1:1775764871.209313:0:6169:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899400. 00010000:00000010:2.1:1775764871.215079:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898e00. 00010000:00000010:3.1:1775764871.228475:0:31:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00010000:00000010:2.1:1775764871.246145:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c000. 00010000:00000010:0.1:1775764871.252345:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6e00. 00000080:00000010:1.1:1775764871.265733:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b8c0. 00010000:00000010:0.1:1775764871.276299:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7a00. 00010000:00000010:2.1:1775764871.290205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1400. 00010000:00000010:3.1:1775764871.314304:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207976900. 00010000:00000010:3.1:1775764871.314318:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ce0a00. 00010000:00000010:3.1:1775764871.326696:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994218fe6400. 00010000:00000010:3.1:1775764871.326711:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1600. 00010000:00000010:3.1:1775764871.338200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c000. 00010000:00000010:2.1:1775764871.355782:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00000080:00000010:3.1:1775764871.365279:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8260. 00010000:00000010:1.1:1775764871.370532:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898e00. 00000080:00000010:3.1:1775764871.378197:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a780. 00010000:00000010:0.1:1775764871.400998:0:2393:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943d00. 00010000:00000010:0.1:1775764871.401014:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1800. 00010000:00000010:2.1:1775764871.407464:0:6168:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00010000:00000010:2.1:1775764871.423285:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d0b600. 00010000:00000010:2.1:1775764871.423295:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d600. 00010000:00000010:2.1:1775764871.423300:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899400. 00000080:00000010:0.1:1775764871.435329:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b440. 00010000:00000010:2.1:1775764871.451670:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898e00. 00010000:00000010:2.1:1775764871.460701:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7000. 00000080:00000010:3.1:1775764871.466398:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b4c0. 00010000:00000010:2.1:1775764871.473409:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994220010300. 00010000:00000010:2.1:1775764871.473419:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6c00. 00010000:00000010:1.1:1775764871.482289:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898e00. 00010000:00000010:2.1:1775764871.498894:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207de4700. 00010000:00000010:2.1:1775764871.498903:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7400. 00010000:00000010:2.1:1775764871.506991:0:14464:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7a00. 00010000:00000010:0.1:1775764871.521926:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899400. 00000080:00000010:3.1:1775764871.539952:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c83c0. 00010000:00000010:1.1:1775764871.563338:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00010000:00000010:3.1:1775764871.570387:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c000. 00000080:00000010:3.1:1775764871.582544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db23e0. 00010000:00000010:2.1:1775764871.590216:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d0b600. 00010000:00000010:2.1:1775764871.590231:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d600. 00010000:00000010:2.1:1775764871.598205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0a00. 00000080:00000010:1.1:1775764871.652217:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2700. 00010000:00000010:2.1:1775764871.652299:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d600. 00000080:00000010:2.1:1775764871.652307:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2ee0. 00000080:00000010:1.1:1775764871.658823:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b180. 00010000:00000010:1.1:1775764871.661391:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1800. 00010000:00000010:1.1:1775764871.661411:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942079b7000. 00010000:00000010:1.1:1775764871.661417:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7600. 00010000:00000010:1.1:1775764871.661421:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899400. 00000080:00000010:0.1:1775764871.665628:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6aa40. 00000080:00000010:0.1:1775764871.665659:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff99422762a940. 00010000:00000010:0.1:1775764871.688407:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6c00. 00010000:00000010:0.1:1775764871.688429:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209d57500. 00010000:00000010:0.1:1775764871.688435:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6a00. 00000080:00000010:0.1:1775764871.706190:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8e80. 00000080:00000010:0.1:1775764871.706225:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994218997380. 00010000:00000010:0.1:1775764871.717245:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6c00. 00000080:00000010:2.1:1775764871.735329:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a900. 00000080:00000010:2.1:1775764871.745300:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8720. 00010000:00000010:2.1:1775764871.745312:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789c000. 00010000:00000010:2.1:1775764871.745319:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942420d9800. 00010000:00000010:2.1:1775764871.745324:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc000. 00010000:00000010:2.1:1775764871.745327:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6a00. 00000080:00000010:3.1:1775764871.746718:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b4c0. 00010000:00000010:3.1:1775764871.746768:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d600. 00010000:00000010:3.1:1775764871.746775:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078e9600. 00010000:00000010:3.1:1775764871.746783:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6e00. 00010000:00000010:3.1:1775764871.746786:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c000. 00000080:00000010:3.1:1775764871.790640:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2ee0. 00000080:00000010:2.1:1775764871.794260:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2ca0. 00000080:00000010:2.1:1775764871.794304:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff99422762f380. 00010000:00000010:2.1:1775764871.794312:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420794cb00. 00010000:00000010:2.1:1775764871.794319:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895e00. 00000080:00000010:0.1:1775764871.794389:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994218996b40. 00010000:00000010:0.1:1775764871.794399:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7c00. 00010000:00000010:0.1:1775764871.794405:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207976200. 00010000:00000010:0.1:1775764871.794409:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798de00. 00000080:00000010:3.1:1775764871.800319:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8260. 00000080:00000010:2.1:1775764871.818301:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2880. 00000080:00000010:3.1:1775764871.818973:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d39c0. 00000080:00000010:2.1:1775764871.822868:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d5280. 00010000:00000010:2.1:1775764871.832387:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078b3600. 00010000:00000010:2.1:1775764871.832395:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd600. 00010000:00000010:3.1:1775764871.835317:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00000080:00000010:2.1:1775764871.848199:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2620. 00000080:00000010:2.1:1775764871.848229:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d1080. 00010000:00000010:2.1:1775764871.859895:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207c67900. 00010000:00000010:2.1:1775764871.859913:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789d400. 00000080:00000010:1.1:1775764871.865855:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b4c0. 00010000:00000010:1.1:1775764871.865867:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789dc00. 00010000:00000010:1.1:1775764871.865873:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420794c700. 00010000:00000010:1.1:1775764871.865878:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899e00. 00010000:00000010:1.1:1775764871.865882:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899400. 00000080:00000010:0.1:1775764871.876256:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2580. 00000080:00000010:2.1:1775764871.888662:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2dc0. 00000080:00000010:1.1:1775764871.888758:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994218992100. 00010000:00000010:1.1:1775764871.915986:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420550bf00. 00010000:00000010:1.1:1775764871.916011:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a6800. 00000080:00000010:3.1:1775764871.934452:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff99422762eb40. 00010000:00000010:3.1:1775764871.934466:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899600. 00010000:00000010:3.1:1775764871.934473:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942420d9100. 00010000:00000010:3.1:1775764871.934477:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970600. 00000080:00000010:3.1:1775764871.934483:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8360. 00000080:00000010:0.1:1775764871.935267:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bb80. 00000080:00000010:0.1:1775764871.935290:0:14453:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227602940. 00000080:00000010:3.1:1775764871.954262:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994218991080. 00010000:00000010:0.1:1775764871.959271:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209d57400. 00010000:00000010:0.1:1775764871.959286:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da200. 00000080:00000010:0.1:1775764871.959292:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2280. 00000080:00000010:0.1:1775764871.959321:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994218994200. 00010000:00000010:2.1:1775764871.964105:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209d57800. 00010000:00000010:2.1:1775764871.964125:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dac00. 00010000:00000010:1.1:1775764871.980068:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078b3000. 00010000:00000010:1.1:1775764871.980083:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dbe00. 00000080:00000010:1.1:1775764871.987913:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bf60. 00000080:00000010:2.1:1775764871.988458:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6be20. 00000080:00000010:2.1:1775764871.988484:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994218995280. 00010000:00000010:2.1:1775764872.011218:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798de00. 00000080:00000010:2.1:1775764872.011228:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2dc0. 00000080:00000010:0.1:1775764872.011314:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d18c0. 00010000:00000010:0.1:1775764872.014398:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1200. 00010000:00000010:0.1:1775764872.014420:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942420d9300. 00010000:00000010:0.1:1775764872.014425:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970400. 00000080:00000010:3.1:1775764872.038283:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2ec0. 00000080:00000010:3.1:1775764872.048458:0:14453:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d2940. 00010000:00000010:3.1:1775764872.048475:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209d57900. 00010000:00000010:3.1:1775764872.048479:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793d000. 00000080:00000010:3.1:1775764872.070234:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8020. 00010000:00000010:1.1:1775764872.070297:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798de00. 00010000:00000010:2.1:1775764872.074307:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1e00. 00010000:00000010:2.1:1775764872.074323:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420788ec00. 00010000:00000010:2.1:1775764872.074327:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789d000. 00010000:00000010:2.1:1775764872.074330:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c000. 00000080:00000010:3.1:1775764872.074434:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994218990840. 00000080:00000010:2.1:1775764872.087415:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2dc0. 00000080:00000010:2.1:1775764872.089930:0:14454:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994218995ac0. 00010000:00000010:3.1:1775764872.094818:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d600. 00010000:00000010:0.1:1775764872.101231:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899600. 00000080:00000010:0.1:1775764872.101240:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b840. 00010000:00000010:2.1:1775764872.101619:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80600. 00010000:00000010:2.1:1775764872.101628:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943f00. 00010000:00000010:2.1:1775764872.101632:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81000. 00000080:00000010:3.1:1775764872.104193:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2620. 00000080:00000010:3.1:1775764872.108376:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994218994a40. 00010000:00000010:3.1:1775764872.119301:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00000080:00000010:3.1:1775764872.119310:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b7e0. 00010000:00000010:0.1:1775764872.128672:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c000. 00010000:00000010:2.1:1775764872.142704:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0200. 00010000:00000010:2.1:1775764872.142713:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943600. 00010000:00000010:2.1:1775764872.142717:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902e00. 00010000:00000010:2.1:1775764872.142723:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7c00. 00000080:00000010:2.1:1775764872.142729:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2dc0. 00000080:00000010:2.1:1775764872.150753:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff99422762e300. 00010000:00000010:2.1:1775764872.167435:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078efc00. 00010000:00000010:2.1:1775764872.167443:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916200. 00010000:00000010:2.1:1775764872.167446:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899600. 00010000:00000010:3.1:1775764872.185216:0:14464:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420387c600. 00010000:00000010:3.1:1775764872.185223:0:14464:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a6200. 00010000:00000010:3.1:1775764872.185227:0:14464:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966400. 00010000:00000010:3.1:1775764872.185230:0:14464:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8200. 00010000:00000010:1.1:1775764872.187806:0:14461:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c000. 00010000:00000010:0.1:1775764872.197411:0:2393:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d22500. 00010000:00000010:0.1:1775764872.197422:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1c00. 00010000:00000010:2.1:1775764872.201450:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903a00. 00010000:00000010:2.1:1775764872.201470:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994205348f00. 00010000:00000010:2.1:1775764872.201475:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81400. 00000080:00000010:2.1:1775764872.201481:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a080. 00000080:00000010:2.1:1775764872.204411:0:6160:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227606300. 00010000:00000010:3.1:1775764872.213471:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99421263ac00. 00010000:00000010:3.1:1775764872.213480:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075ca800. 00010000:00000010:2.1:1775764872.222638:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ef800. 00010000:00000010:2.1:1775764872.222661:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca1200. 00010000:00000010:0.1:1775764872.226476:0:14473:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966700. 00010000:00000010:0.1:1775764872.226495:0:14473:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb9c00. 00010000:00000010:0.1:1775764872.226501:0:14473:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d22a00. 00010000:00000010:0.1:1775764872.226505:0:14473:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2a00. 00010000:00000010:1.1:1775764872.226783:0:6168:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99421835af00. 00010000:00000010:1.1:1775764872.226791:0:6168:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80400. 00010000:00000010:3.1:1775764872.228200:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994220010400. 00010000:00000010:3.1:1775764872.228211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ce0200. 00010000:00000010:3.1:1775764872.228217:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d22000. 00010000:00000010:3.1:1775764872.228224:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1600. 00010000:00000010:3.1:1775764872.228229:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99421263ae00. 00010000:00000010:3.1:1775764872.228235:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793d200. 00000080:00000010:2.1:1775764872.228386:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2880. 00000080:00000010:2.1:1775764872.228416:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227605280. 00010000:00000010:1.1:1775764872.233224:0:6158:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994220010a00. 00010000:00000010:1.1:1775764872.233241:0:6158:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0200. 00010000:00000010:1.1:1775764872.233246:0:6158:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420387c800. 00010000:00000010:1.1:1775764872.233251:0:6158:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a6400. 00010000:00000010:1.1:1775764872.233254:0:6158:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994220010200. 00010000:00000010:1.1:1775764872.233257:0:6158:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7600. 00010000:00000010:1.1:1775764872.233263:0:6158:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d22100. 00010000:00000010:1.1:1775764872.233266:0:6158:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917e00. 00010000:00000010:1.1:1775764872.233270:0:6158:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99421835a100. 00010000:00000010:1.1:1775764872.233274:0:6158:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80000. 00010000:00000010:1.1:1775764872.233301:0:6158:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994220010100. 00010000:00000010:1.1:1775764872.233304:0:6158:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7200. 00010000:00000010:0.1:1775764872.242590:0:14473:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078efd00. 00010000:00000010:0.1:1775764872.242601:0:14473:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075cac00. 00010000:00000010:1.1:1775764872.242904:0:2393:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994220010c00. 00010000:00000010:1.1:1775764872.242912:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7800. 00010000:00000010:1.1:1775764872.242917:0:2393:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420387ca00. 00010000:00000010:1.1:1775764872.242921:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895800. 00010000:00000010:1.1:1775764872.242925:0:2393:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99421835ac00. 00010000:00000010:1.1:1775764872.242928:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81600. 00010000:00000010:1.1:1775764872.242932:0:2393:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966c00. 00010000:00000010:1.1:1775764872.242936:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8000. 00010000:00000010:3.1:1775764872.284184:0:14464:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207de4500. 00010000:00000010:3.1:1775764872.284192:0:14464:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1a00. 00010000:00000010:3.1:1775764872.284197:0:14464:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420387c500. 00010000:00000010:3.1:1775764872.284200:0:14464:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971a00. 00000080:00000010:2.1:1775764872.301187:0:14473:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2320. 00010000:00000010:0.1:1775764872.304585:0:6158:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99421835a600. 00010000:00000010:0.1:1775764872.304595:0:6158:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793d800. 00000080:00000010:2.1:1775764872.314240:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227601080. 00010000:00000010:0.1:1775764872.314340:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99421835ad00. 00010000:00000010:0.1:1775764872.314350:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81800. 00010000:00000010:3.1:1775764872.314587:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207de4900. 00010000:00000010:3.1:1775764872.314595:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1000. 00010000:00000010:3.1:1775764872.314599:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420387c300. 00010000:00000010:3.1:1775764872.314603:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970e00. 00010000:00000010:3.1:1775764872.314608:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966a00. 00010000:00000010:3.1:1775764872.314611:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970200. 00010000:00000010:3.1:1775764872.314614:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99421835a300. 00010000:00000010:3.1:1775764872.314617:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80a00. 00010000:00000010:3.1:1775764872.314620:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994220010900. 00010000:00000010:3.1:1775764872.314623:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6600. 00010000:00000010:1.1:1775764872.319046:0:2393:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99421835a500. 00010000:00000010:1.1:1775764872.319064:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994209db3000. 00010000:00000010:1.1:1775764872.319070:0:2393:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d0bf00. 00010000:00000010:1.1:1775764872.319073:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903c00. 00010000:00000010:2.1:1775764872.345183:0:14473:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d600. 00000080:00000010:2.1:1775764872.345193:0:14473:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2dc0. 00010000:00000010:2.1:1775764872.345209:0:14473:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207de4300. 00010000:00000010:2.1:1775764872.345215:0:14473:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898400. 00010000:00000010:2.1:1775764872.345221:0:14473:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d0b100. 00010000:00000010:2.1:1775764872.345228:0:14473:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6400. 00010000:00000010:3.1:1775764872.345262:0:6160:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420387c900. 00010000:00000010:3.1:1775764872.345268:0:6160:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894600. 00010000:00000010:3.1:1775764872.356188:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207de4100. 00010000:00000010:3.1:1775764872.356196:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899a00. 00010000:00000010:1.1:1775764872.357553:0:2400:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207de4e00. 00010000:00000010:1.1:1775764872.357561:0:2400:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899200. 00010000:00000010:2.1:1775764872.370255:0:2403:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420794cc00. 00010000:00000010:2.1:1775764872.370283:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894a00. 00010000:00000010:2.1:1775764872.370288:0:2403:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207de4800. 00010000:00000010:2.1:1775764872.370292:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898c00. 00010000:00000010:2.1:1775764872.370295:0:2403:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943500. 00010000:00000010:2.1:1775764872.370299:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1600. 00010000:00000010:2.1:1775764872.370302:0:2403:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207de4600. 00010000:00000010:2.1:1775764872.370306:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789da00. 00010000:00000010:2.1:1775764872.370311:0:2403:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994210812000. 00010000:00000010:2.1:1775764872.370315:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1200. 00010000:00000010:2.1:1775764872.370321:0:2403:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207de4200. 00010000:00000010:2.1:1775764872.370325:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902400. 00010000:00000010:2.1:1775764872.370331:0:2403:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d0bd00. 00010000:00000010:2.1:1775764872.370333:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0e00. 00010000:00000010:3.1:1775764872.376393:0:32:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99421835ab00. 00010000:00000010:3.1:1775764872.376411:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1800. 00010000:00000010:3.1:1775764872.376416:0:32:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994218fe6200. 00010000:00000010:3.1:1775764872.376428:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798de00. 00010000:00000010:1.1:1775764872.380355:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207976700. 00010000:00000010:1.1:1775764872.380394:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895400. 00010000:00000010:1.1:1775764872.380398:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d0bc00. 00010000:00000010:1.1:1775764872.380403:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c600. 00010000:00000010:3.1:1775764872.380798:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d0b300. 00010000:00000010:3.1:1775764872.380804:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0600. 00010000:00000010:2.1:1775764872.407800:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ef700. 00010000:00000010:2.1:1775764872.407808:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fcc00. 00000080:00000010:3.1:1775764872.427194:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2420. 00010000:00000010:2.1:1775764872.436593:0:2403:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d0b500. 00010000:00000010:2.1:1775764872.436603:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d400. 00000080:00000010:0.1:1775764872.448378:0:14454:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff99422762b180. 00010000:00000010:0.1:1775764872.498083:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078efa00. 00010000:00000010:0.1:1775764872.498091:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917000. 00010000:00000010:0.1:1775764872.498096:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c000. 00000080:00000010:0.1:1775764872.498099:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8360. 00000080:00000010:0.1:1775764872.498123:0:14453:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942189af380. 00010000:00000010:0.1:1775764872.498129:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7600. 00000080:00000010:2.1:1775764872.498270:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b5a0. 00000080:00000010:2.1:1775764872.525233:0:1358:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942276039c0. 00010000:00000010:1.1:1775764872.526474:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207de4200. 00010000:00000010:1.1:1775764872.526492:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7800. 00000080:00000010:1.1:1775764872.546292:0:20:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c84a0. 00000080:00000010:1.1:1775764872.546302:0:20:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2fc0. 00010000:00000010:1.1:1775764872.546309:0:20:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420794c800. 00010000:00000010:1.1:1775764872.546314:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903200. 00010000:00000010:1.1:1775764872.546320:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d88800. 00010000:00000010:0.1:1775764872.546372:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078e9400. 00010000:00000010:0.1:1775764872.546379:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca0a00. 00000080:00000010:1.1:1775764872.576691:0:2392:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942189aa100. 00000080:00000010:3.1:1775764872.577394:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2160. 00010000:00000010:2.1:1775764872.587377:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d0b500. 00010000:00000010:2.1:1775764872.587394:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0200. 00000080:00000010:0.1:1775764872.589141:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6abc0. 00000080:00000010:0.1:1775764872.632305:0:14453:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d5ac0. 00000080:00000010:3.1:1775764872.632543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a780. 00000080:00000010:3.1:1775764872.632563:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227602100. 00010000:00000010:0.1:1775764872.637238:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420b307b00. 00010000:00000010:0.1:1775764872.637257:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db200. 00010000:00000010:2.1:1775764872.641821:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994218fe6200. 00010000:00000010:2.1:1775764872.641844:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899a00. 00000080:00000010:2.1:1775764872.650685:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b140. 00000080:00000010:2.1:1775764872.657233:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d9080. 00000080:00000010:2.1:1775764872.681194:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b680. 00010000:00000010:3.1:1775764872.707206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d400. 00000080:00000010:2.1:1775764872.707286:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275dca40. 00000080:00000010:3.1:1775764872.711375:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2dc0. 00000080:00000010:3.1:1775764872.711418:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2560. 00010000:00000010:3.1:1775764872.711456:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078a1d00. 00010000:00000010:3.1:1775764872.711464:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902200. 00010000:00000010:3.1:1775764872.711471:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903e00. 00010000:00000010:0.1:1775764872.711558:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942079b7300. 00010000:00000010:0.1:1775764872.711563:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902c00. 00000080:00000010:3.1:1775764872.726855:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2e00. 00000080:00000010:1.1:1775764872.785791:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2960. 00000080:00000010:3.1:1775764872.818242:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d6300. 00000080:00000010:1.1:1775764872.843067:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d4200. 00010000:00000010:1.1:1775764872.843090:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994205348a00. 00010000:00000010:1.1:1775764872.843097:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7e00. 00010000:00000010:3.1:1775764872.843207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00010000:00000010:2.1:1775764872.843269:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207eef700. 00010000:00000010:2.1:1775764872.843275:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6a00. 00000080:00000010:1.1:1775764872.879260:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2440. 00000080:00000010:3.1:1775764872.879386:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b2e0. 00000080:00000010:3.1:1775764872.884294:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942276298c0. 00000080:00000010:2.1:1775764872.884602:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275db180. 00010000:00000010:3.1:1775764872.887610:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420794c400. 00010000:00000010:3.1:1775764872.887635:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da600. 00010000:00000010:2.1:1775764872.904870:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207976c00. 00010000:00000010:2.1:1775764872.904896:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d89e00. 00000080:00000010:2.1:1775764872.910227:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2b00. 00000080:00000010:3.1:1775764872.910302:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bac0. 00000080:00000010:2.1:1775764872.918759:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942189ae300. 00010000:00000010:1.1:1775764872.940140:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942420d9f00. 00010000:00000010:1.1:1775764872.940178:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994218f6fe00. 00000080:00000010:2.1:1775764872.976265:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8fe0. 00000080:00000010:2.1:1775764872.983368:0:14454:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227605ac0. 00000080:00000010:1.1:1775764872.991287:0:2392:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942189918c0. 00010000:00000010:3.1:1775764873.016427:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d600. 00010000:00000010:0.1:1775764873.042833:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207eefb00. 00010000:00000010:0.1:1775764873.042843:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db400. 00000080:00000010:2.1:1775764873.062243:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b540. 00000080:00000010:0.1:1775764873.062713:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db23c0. 00000080:00000010:0.1:1775764873.062744:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d6b40. 00010000:00000010:0.1:1775764873.062751:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207976400. 00010000:00000010:0.1:1775764873.062757:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d88c00. 00000080:00000010:1.1:1775764873.066793:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2b00. 00000080:00000010:1.1:1775764873.066817:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8b60. 00000080:00000010:1.1:1775764873.066823:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2220. 00000080:00000010:1.1:1775764873.066826:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2e40. 00000080:00000010:1.1:1775764873.066832:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b760. 00000080:00000010:1.1:1775764873.066840:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db22c0. 00000080:00000010:1.1:1775764873.066848:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b5c0. 00000080:00000010:1.1:1775764873.066852:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b660. 00000080:00000010:1.1:1775764873.066856:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2040. 00000080:00000010:1.1:1775764873.066859:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2b20. 00000080:00000010:1.1:1775764873.066864:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2540. 00000080:00000010:1.1:1775764873.066867:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2300. 00000080:00000010:1.1:1775764873.066873:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db27a0. 00000080:00000010:1.1:1775764873.066878:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2be0. 00000080:00000010:1.1:1775764873.066882:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b340. 00000080:00000010:1.1:1775764873.066886:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c83a0. 00000080:00000010:1.1:1775764873.066890:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a660. 00010000:00000010:1.1:1775764873.066899:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966000. 00010000:00000010:1.1:1775764873.066905:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075dbe00. 00010000:00000010:1.1:1775764873.066910:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d200. 00000080:00000010:2.1:1775764873.076218:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275deb40. 00000080:00000010:0.1:1775764873.082538:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2940. 00000080:00000010:1.1:1775764873.085697:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff99422762b9c0. 00010000:00000010:2.1:1775764873.095807:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d600. 00000080:00000010:2.1:1775764873.095822:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2320. 00010000:00000010:2.1:1775764873.103556:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943800. 00010000:00000010:2.1:1775764873.103574:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7c00. 00000080:00000010:3.1:1775764873.111675:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b1a0. 00000080:00000010:3.1:1775764873.112001:0:14454:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d3180. 00010000:00000010:2.1:1775764873.128462:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00000080:00000010:2.1:1775764873.159275:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bac0. 00000080:00000010:2.1:1775764873.159288:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db26c0. 00000080:00000010:2.1:1775764873.159295:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b8a0. 00000080:00000010:2.1:1775764873.159299:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b080. 00000080:00000010:2.1:1775764873.159302:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2180. 00000080:00000010:2.1:1775764873.159307:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db28e0. 00000080:00000010:2.1:1775764873.159311:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a340. 00000080:00000010:2.1:1775764873.159316:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a880. 00000080:00000010:2.1:1775764873.159321:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8660. 00000080:00000010:2.1:1775764873.159325:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2100. 00000080:00000010:2.1:1775764873.159332:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db29a0. 00000080:00000010:2.1:1775764873.159336:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2f80. 00000080:00000010:2.1:1775764873.159340:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b000. 00000080:00000010:2.1:1775764873.159344:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2680. 00000080:00000010:2.1:1775764873.159347:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db21a0. 00000080:00000010:2.1:1775764873.159351:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2b60. 00000080:00000010:2.1:1775764873.159366:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2e60. 00010000:00000010:2.1:1775764873.159374:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994206989500. 00010000:00000010:2.1:1775764873.159380:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971000. 00010000:00000010:2.1:1775764873.159385:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894400. 00010000:00000010:2.1:1775764873.159388:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899400. 00000080:00000010:3.1:1775764873.178800:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8860. 00000080:00000010:1.1:1775764873.188980:0:2392:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227618000. 00010000:00000010:2.1:1775764873.189228:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994203fb0200. 00010000:00000010:2.1:1775764873.189237:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ad900. 00010000:00000010:2.1:1775764873.189242:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1a00. 00010000:00000010:2.1:1775764873.189246:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d400. 00000080:00000010:2.1:1775764873.189252:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b1a0. 00000080:00000010:2.1:1775764873.189267:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994218992940. 00010000:00000010:3.1:1775764873.195223:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971c00. 00010000:00000010:3.1:1775764873.195233:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943200. 00010000:00000010:3.1:1775764873.195237:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902a00. 00010000:00000010:3.1:1775764873.203203:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0600. 00000080:00000010:3.1:1775764873.203214:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a880. 00000080:00000010:0.1:1775764873.211559:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff99420756fda0. 00000080:00000010:0.1:1775764873.211586:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994218996300. 00010000:00000010:2.1:1775764873.243320:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6000. 00010000:00000010:3.1:1775764873.244210:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942069a5a00. 00010000:00000010:3.1:1775764873.244221:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc400. 00000080:00000010:3.1:1775764873.264775:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8860. 00000080:00000010:3.1:1775764873.264816:0:14454:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994218990000. 00010000:00000010:0.1:1775764873.273199:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8800. 00010000:00000010:0.1:1775764873.273207:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943700. 00010000:00000010:0.1:1775764873.273211:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916600. 00010000:00000010:2.1:1775764873.274444:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d400. 00010000:00000010:2.1:1775764873.274449:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079ca200. 00010000:00000010:2.1:1775764873.274452:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966500. 00010000:00000010:2.1:1775764873.274455:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902600. 00010000:00000010:2.1:1775764873.274459:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899e00. 00000080:00000010:1.1:1775764873.286200:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2e60. 00000080:00000010:3.1:1775764873.286433:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8620. 00000080:00000010:0.1:1775764873.289847:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d7380. 00000080:00000010:2.1:1775764873.290240:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d2100. 00010000:00000010:0.1:1775764873.294552:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971400. 00010000:00000010:0.1:1775764873.294562:0:2392:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966300. 00010000:00000010:0.1:1775764873.294577:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970a00. 00010000:00000010:3.1:1775764873.295200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d600. 00000080:00000010:3.1:1775764873.295209:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2380. 00000080:00000010:2.1:1775764873.306912:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2060. 00000080:00000010:2.1:1775764873.306954:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d4a40. 00010000:00000010:3.1:1775764873.326209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d400. 00010000:00000010:1.1:1775764873.336190:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ad100. 00010000:00000010:1.1:1775764873.336211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0400. 00010000:00000010:0.1:1775764873.336602:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420788ed00. 00010000:00000010:0.1:1775764873.336609:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d88e00. 00010000:00000010:0.1:1775764873.336613:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899e00. 00000080:00000010:0.1:1775764873.361209:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2b60. 00000080:00000010:0.1:1775764873.361235:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942189939c0. 00000080:00000010:2.1:1775764873.366892:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bae0. 00000080:00000010:2.1:1775764873.366929:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942189ab9c0. 00010000:00000010:2.1:1775764873.379223:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420788e900. 00010000:00000010:2.1:1775764873.379243:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798da00. 00010000:00000010:1.1:1775764873.379298:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d000. 00010000:00000010:1.1:1775764873.379305:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942069a5400. 00010000:00000010:1.1:1775764873.379309:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0e00. 00000080:00000010:2.1:1775764873.398991:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8640. 00000080:00000010:2.1:1775764873.399026:0:14454:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227629080. 00010000:00000010:1.1:1775764873.403951:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7400. 00000080:00000010:1.1:1775764873.412060:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c83a0. 00000080:00000010:2.1:1775764873.412196:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db26a0. 00000080:00000010:2.1:1775764873.412223:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942189a8000. 00010000:00000010:0.1:1775764873.421218:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078e9500. 00010000:00000010:0.1:1775764873.421230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916400. 00000080:00000010:2.1:1775764873.454852:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a5c0. 00000080:00000010:2.1:1775764873.454901:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff99422762a100. 00010000:00000010:0.1:1775764873.487615:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943100. 00010000:00000010:0.1:1775764873.487636:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db600. 00010000:00000010:0.1:1775764873.487643:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6600. 00000080:00000010:0.1:1775764873.487673:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bf20. 00000080:00000010:0.1:1775764873.487711:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275df380. 00010000:00000010:3.1:1775764873.488714:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798da00. 00000080:00000010:3.1:1775764873.488726:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db26a0. 00010000:00000010:0.1:1775764873.502895:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209897c00. 00010000:00000010:0.1:1775764873.502916:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cbd600. 00000080:00000010:2.1:1775764873.507321:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2c20. 00000080:00000010:2.1:1775764873.507363:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d0000. 00010000:00000010:2.1:1775764873.507375:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207c70100. 00010000:00000010:2.1:1775764873.507381:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971600. 00000080:00000010:0.1:1775764873.526209:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b0e0. 00000080:00000010:1.1:1775764873.552193:0:14454:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d0840. 00010000:00000010:3.1:1775764873.574223:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c000. 00010000:00000010:3.1:1775764873.582239:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c600. 00010000:00000010:2.1:1775764873.596380:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798de00. 00000080:00000010:2.1:1775764873.596401:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2060. 00010000:00000010:3.1:1775764873.634903:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1400. 00010000:00000010:3.1:1775764873.667388:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6e00. 00010000:00000010:0.1:1775764873.712770:0:2401:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798de00. 00010000:00000010:3.1:1775764873.728851:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00000080:00000010:2.1:1775764873.739184:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2c20. 00010000:00000010:1.1:1775764873.782283:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c000. 00010000:00000010:2.1:1775764873.799255:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917600. 00010000:00000010:3.1:1775764873.807304:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7000. 00010000:00000010:2.1:1775764873.835203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00010000:00000010:3.1:1775764873.865695:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798da00. 00000080:00000010:3.1:1775764873.877402:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2060. 00000080:00000010:0.1:1775764873.887225:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8020. 00010000:00000010:0.1:1775764873.887246:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d000. 00010000:00000010:1.1:1775764873.889279:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6a00. 00010000:00000010:3.1:1775764873.928291:0:14464:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898c00. 00010000:00000010:3.1:1775764873.940197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899a00. 00000080:00000010:2.1:1775764873.941333:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2680. 00010000:00000010:3.1:1775764873.978888:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d400. 00010000:00000010:3.1:1775764874.002507:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d600. 00000080:00000010:2.1:1775764874.018189:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a340. 00010000:00000010:2.1:1775764874.018204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d200. 00010000:00000010:3.1:1775764874.018638:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1000. 00010000:00000010:3.1:1775764874.034364:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d000. 00000080:00000010:2.1:1775764874.046251:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b2e0. 00010000:00000010:3.1:1775764874.050257:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1e00. 00000080:00000010:1.1:1775764874.056572:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a340. 00010000:00000010:1.1:1775764874.056590:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899a00. 00010000:00000010:2.1:1775764874.064193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d200. 00000080:00000010:3.1:1775764874.067576:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2c20. 00010000:00000010:2.1:1775764874.101195:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798da00. 00010000:00000010:3.1:1775764874.122922:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898c00. 00010000:00000010:2.1:1775764874.155971:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d200. 00010000:00000010:0.1:1775764874.165230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898c00. 00000080:00000010:3.1:1775764874.169395:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b220. 00010000:00000010:3.1:1775764874.169411:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c000. 00000080:00000010:3.1:1775764874.169417:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2680. 00010000:00000010:2.1:1775764874.169487:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798de00. 00010000:00000010:2.1:1775764874.175749:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899400. 00010000:00000010:3.1:1775764874.178443:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898400. 00010000:00000010:2.1:1775764874.200287:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6c00. 00010000:00000010:2.1:1775764874.209286:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d0b500. 00010000:00000010:2.1:1775764874.209296:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d200. 00000080:00000010:2.1:1775764874.237208:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2100. 00000080:00000010:2.1:1775764874.237232:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275bca40. 00010000:00000010:1.1:1775764874.249526:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c000. 00000080:00000010:1.1:1775764874.249538:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2180. 00010000:00000010:3.1:1775764874.254236:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d200. 00010000:00000010:2.1:1775764874.266150:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c600. 00010000:00000010:2.1:1775764874.274170:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d000. 00010000:00000010:2.1:1775764874.287238:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0a00. 00010000:00000010:1.1:1775764874.331278:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d000. 00010000:00000010:2.1:1775764874.364298:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d0b500. 00010000:00000010:2.1:1775764874.364306:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d600. 00010000:00000010:3.1:1775764874.373221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1a00. 00000080:00000010:3.1:1775764874.373229:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a820. 00000080:00000010:1.1:1775764874.391104:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db26c0. 00000080:00000010:1.1:1775764874.391127:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275bca40. 00010000:00000010:2.1:1775764874.398229:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d600. 00010000:00000010:2.1:1775764874.402708:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898400. 00010000:00000010:2.1:1775764874.435595:0:2403:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994218fe6800. 00010000:00000010:2.1:1775764874.435609:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d400. 00010000:00000010:2.1:1775764874.453182:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1600. 00010000:00000010:3.1:1775764874.459328:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899200. 00010000:00000010:3.1:1775764874.516057:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d0b500. 00010000:00000010:3.1:1775764874.516079:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c000. 00010000:00000010:0.1:1775764874.553619:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899200. 00010000:00000010:3.1:1775764874.557256:0:14464:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078e9b00. 00010000:00000010:3.1:1775764874.557266:0:14464:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075da800. 00010000:00000010:0.1:1775764874.558313:0:6159:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207976300. 00010000:00000010:0.1:1775764874.558322:0:6159:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1000. 00010000:00000010:0.1:1775764874.558325:0:6159:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994210a04a00. 00010000:00000010:0.1:1775764874.558329:0:6159:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80200. 00010000:00000010:2.1:1775764874.558625:0:6168:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966600. 00010000:00000010:2.1:1775764874.558632:0:6168:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994203fb1600. 00010000:00000010:2.1:1775764874.558636:0:6168:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209d57700. 00010000:00000010:2.1:1775764874.558639:0:6168:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971e00. 00010000:00000010:2.1:1775764874.558644:0:6168:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943e00. 00010000:00000010:2.1:1775764874.558647:0:6168:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903600. 00010000:00000010:2.1:1775764874.558652:0:6168:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207c67500. 00010000:00000010:2.1:1775764874.558655:0:6168:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903400. 00010000:00000010:3.1:1775764874.570240:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ad600. 00010000:00000010:3.1:1775764874.570256:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902000. 00010000:00000010:3.1:1775764874.574370:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207976800. 00010000:00000010:3.1:1775764874.574392:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0000. 00010000:00000010:3.1:1775764874.574397:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994206989300. 00010000:00000010:3.1:1775764874.574401:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916800. 00010000:00000010:3.1:1775764874.574404:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ad500. 00010000:00000010:3.1:1775764874.574408:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7a00. 00010000:00000010:2.1:1775764874.574477:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420794c300. 00010000:00000010:2.1:1775764874.574484:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb3e00. 00010000:00000010:2.1:1775764874.574490:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420788e100. 00010000:00000010:2.1:1775764874.574494:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80800. 00010000:00000010:2.1:1775764874.574497:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994210812900. 00010000:00000010:2.1:1775764874.574501:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420694d400. 00010000:00000010:2.1:1775764874.574517:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966200. 00010000:00000010:2.1:1775764874.574521:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970800. 00010000:00000010:2.1:1775764874.574525:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078adc00. 00010000:00000010:2.1:1775764874.574528:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb3600. 00010000:00000010:1.1:1775764874.625682:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d0ba00. 00010000:00000010:1.1:1775764874.625702:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798de00. 00010000:00000010:1.1:1775764874.643211:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d0b000. 00010000:00000010:1.1:1775764874.643221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c600. 00000080:00000010:2.1:1775764874.788366:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db25c0. 00010000:00000010:2.1:1775764874.812461:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0800. 00010000:00000010:0.1:1775764874.823310:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d400. 00010000:00000010:3.1:1775764874.864327:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6000. 00010000:00000010:0.1:1775764874.895859:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0400. 00000080:00000010:3.1:1775764874.899027:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a340. 00010000:00000010:3.1:1775764874.899055:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899600. 00000080:00000010:2.1:1775764874.956246:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2100. 00010000:00000010:0.1:1775764874.964184:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899200. 00010000:00000010:2.1:1775764874.984202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d400. 00010000:00000010:3.1:1775764875.005214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c600. 00000080:00000010:3.1:1775764875.034180:0:1026:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2d80. 00010000:00000010:3.1:1775764875.047918:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7c00. 00010000:00000010:0.1:1775764875.071014:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0a00. 00010000:00000010:2.1:1775764875.076347:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798de00. 00000080:00000010:0.1:1775764875.094385:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db25c0. 00010000:00000010:3.1:1775764875.100194:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7a00. 00010000:00000010:2.1:1775764875.123214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d400. 00010000:00000010:0.1:1775764875.143072:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1c00. 00000080:00000010:2.1:1775764875.148191:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b120. 00010000:00000010:3.1:1775764875.163213:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d400. 00010000:00000010:3.1:1775764875.170512:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798de00. 00010000:00000010:3.1:1775764875.180363:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898400. 00010000:00000010:2.1:1775764875.194396:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898c00. 00000080:00000010:3.1:1775764875.195127:0:14464:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c83a0. 00010000:00000010:3.1:1775764875.206786:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c000. 00010000:00000010:3.1:1775764875.206793:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899400. 00010000:00000010:2.1:1775764875.221455:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798de00. 00000080:00000010:2.1:1775764875.226414:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2d60. 00010000:00000010:3.1:1775764875.240516:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798de00. 00000080:00000010:2.1:1775764875.271180:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b560. 00010000:00000010:3.1:1775764875.278933:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d000. 00010000:00000010:3.1:1775764875.281269:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c600. 00010000:00000010:1.1:1775764875.292376:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d400. 00010000:00000010:1.1:1775764875.292385:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7a00. 00010000:00000010:2.1:1775764875.307205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7600. 00000080:00000010:2.1:1775764875.330383:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2360. 00010000:00000010:3.1:1775764875.335460:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6c00. 00000080:00000010:0.1:1775764875.336175:0:14473:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8980. 00010000:00000010:0.1:1775764875.336191:0:14473:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7c00. 00010000:00000010:2.1:1775764875.341238:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c000. 00010000:00000010:2.1:1775764875.361214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798de00. 00010000:00000010:2.1:1775764875.374483:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d400. 00010000:00000010:2.1:1775764875.384621:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0000. 00000080:00000010:2.1:1775764875.384639:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a580. 00010000:00000010:2.1:1775764875.417293:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798de00. 00010000:00000010:2.1:1775764875.417300:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d400. 00000080:00000010:3.1:1775764875.418221:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db20a0. 00010000:00000010:3.1:1775764875.418236:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c000. 00010000:00000010:2.1:1775764875.441188:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d400. 00000080:00000010:2.1:1775764875.451444:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2120. 00010000:00000010:3.1:1775764875.477650:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798de00. 00000080:00000010:1.1:1775764875.484481:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2d60. 00000080:00000010:1.1:1775764875.490194:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b900. 00010000:00000010:1.1:1775764875.503431:0:2393:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d0bb00. 00010000:00000010:1.1:1775764875.503446:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c600. 00000080:00000010:2.1:1775764875.508903:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2900. 00000080:00000010:2.1:1775764875.508929:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff99422762b180. 00000080:00000010:3.1:1775764875.524198:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2ea0. 00000080:00000010:3.1:1775764875.544530:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b940. 00010000:00000010:3.1:1775764875.565112:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d0b000. 00010000:00000010:3.1:1775764875.565138:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899400. 00010000:00000010:2.1:1775764875.573201:0:6169:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d000. 00000080:00000010:2.1:1775764875.583421:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6baa0. 00000080:00000010:2.1:1775764875.583443:0:14453:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275bd280. 00010000:00000010:2.1:1775764875.596685:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c600. 00000080:00000010:2.1:1775764875.611646:0:6168:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db28a0. 00010000:00000010:3.1:1775764875.621293:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c000. 00010000:00000010:2.1:1775764875.628319:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c600. 00010000:00000010:2.1:1775764875.649353:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420694d400. 00000080:00000010:2.1:1775764875.649380:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db28a0. 00010000:00000010:2.1:1775764875.662187:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d000. 00000080:00000010:2.1:1775764875.680003:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db22a0. 00010000:00000010:0.1:1775764875.688460:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420694d400. 00010000:00000010:2.1:1775764875.689384:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994203fb0200. 00010000:00000010:2.1:1775764875.689431:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798de00. 00010000:00000010:2.1:1775764875.689435:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994203fb1600. 00010000:00000010:3.1:1775764875.695385:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902000. 00000080:00000010:3.1:1775764875.695406:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2240. 00000080:00000010:2.1:1775764875.715298:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db28a0. 00000080:00000010:2.1:1775764875.715323:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2900. 00010000:00000010:2.1:1775764875.715330:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903400. 00000080:00000010:2.1:1775764875.715347:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db22a0. 00010000:00000010:3.1:1775764875.753274:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898400. 00010000:00000010:3.1:1775764875.753326:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903600. 00010000:00000010:2.1:1775764875.758292:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903400. 00010000:00000010:2.1:1775764875.763271:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0600. 00010000:00000010:2.1:1775764875.763283:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899400. 00000080:00000010:2.1:1775764875.763289:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db22a0. 00000080:00000010:2.1:1775764875.763296:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db24e0. 00010000:00000010:2.1:1775764875.785229:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902600. 00000080:00000010:0.1:1775764875.785483:0:2392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2900. 00010000:00000010:3.1:1775764875.803427:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7800. 00010000:00000010:2.1:1775764875.835408:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902600. 00010000:00000010:2.1:1775764875.835455:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7200. 00010000:00000010:3.1:1775764875.867188:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207926200. 00010000:00000010:3.1:1775764875.867195:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899a00. 00010000:00000010:2.1:1775764875.886207:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902a00. 00010000:00000010:1.1:1775764875.903523:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902600. 00010000:00000010:3.1:1775764875.926392:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d0bd00. 00010000:00000010:3.1:1775764875.926416:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903400. 00010000:00000010:2.1:1775764875.943313:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902a00. 00010000:00000010:2.1:1775764875.981221:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d0ba00. 00010000:00000010:2.1:1775764875.981246:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d400. 00010000:00000010:2.1:1775764876.013289:0:2402:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943e00. 00010000:00000010:2.1:1775764876.013315:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0200. 00000080:00000010:2.1:1775764876.179248:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6aac0. 00010000:00000010:2.1:1775764876.183286:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0800. 00010000:00000010:2.1:1775764876.183293:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903e00. 00010000:00000010:0.1:1775764876.193939:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7200. 00000080:00000010:2.1:1775764876.196450:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c83a0. 00010000:00000010:2.1:1775764876.196485:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902200. 00010000:00000010:3.1:1775764876.211357:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899a00. 00010000:00000010:2.1:1775764876.237495:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898400. 00000080:00000010:2.1:1775764876.296189:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db24e0. 00000080:00000010:0.1:1775764876.323284:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b940. 00010000:00000010:3.1:1775764876.350417:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902200. 00010000:00000010:2.1:1775764876.359657:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6200. 00010000:00000010:3.1:1775764876.384372:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902400. 00010000:00000010:2.1:1775764876.391578:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903200. 00010000:00000010:3.1:1775764876.444082:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903200. 00000080:00000010:2.1:1775764876.447878:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2bc0. 00010000:00000010:2.1:1775764876.447913:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903c00. 00010000:00000010:2.1:1775764876.459220:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0200. 00000080:00000010:1.1:1775764876.462352:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a280. 00010000:00000010:1.1:1775764876.462386:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903a00. 00000080:00000010:3.1:1775764876.518728:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2480. 00010000:00000010:3.1:1775764876.543258:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899a00. 00000080:00000010:2.1:1775764876.543336:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8320. 00010000:00000010:2.1:1775764876.585272:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903c00. 00010000:00000010:3.1:1775764876.588212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00010000:00000010:2.1:1775764876.645184:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0400. 00010000:00000010:3.1:1775764876.675948:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1600. 00010000:00000010:2.1:1775764876.678273:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00010000:00000010:2.1:1775764876.713212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903c00. 00010000:00000010:2.1:1775764876.713226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0400. 00000080:00000010:1.1:1775764876.732114:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6afc0. 00000080:00000010:2.1:1775764876.747556:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b1e0. 00010000:00000010:0.1:1775764876.760261:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903c00. 00010000:00000010:0.1:1775764876.760281:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1e00. 00010000:00000010:2.1:1775764876.765145:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902e00. 00010000:00000010:2.1:1775764876.765316:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942069a5a00. 00010000:00000010:2.1:1775764876.765333:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899600. 00010000:00000010:1.1:1775764876.783203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0800. 00010000:00000010:1.1:1775764876.783211:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943a00. 00010000:00000010:1.1:1775764876.783215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1000. 00000080:00000010:3.1:1775764876.783266:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a900. 00000080:00000010:3.1:1775764876.800211:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275ba940. 00000080:00000010:2.1:1775764876.800592:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2bc0. 00000080:00000010:2.1:1775764876.800614:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275b8840. 00010000:00000010:3.1:1775764876.845200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1000. 00010000:00000010:3.1:1775764876.845219:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902e00. 00000080:00000010:2.1:1775764876.846104:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db24e0. 00010000:00000010:0.1:1775764876.855630:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903c00. 00000080:00000010:2.1:1775764876.868239:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6be00. 00010000:00000010:3.1:1775764876.873896:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1400. 00010000:00000010:0.1:1775764876.875181:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903200. 00010000:00000010:2.1:1775764876.887825:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6600. 00010000:00000010:2.1:1775764876.904795:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903a00. 00000080:00000010:1.1:1775764876.913007:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2f60. 00010000:00000010:2.1:1775764876.937622:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902400. 00010000:00000010:1.1:1775764876.944186:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902200. 00010000:00000010:3.1:1775764876.946664:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0200. 00010000:00000010:2.1:1775764876.960882:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1400. 00000080:00000010:0.1:1775764876.991581:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6aac0. 00000080:00000010:3.1:1775764876.997851:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db24e0. 00010000:00000010:0.1:1775764877.019615:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7400. 00010000:00000010:2.1:1775764877.022916:0:1026:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899e00. 00010000:00000010:2.1:1775764877.022932:0:1026:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0600. 00010000:00000010:0.1:1775764877.027625:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899a00. 00010000:00000010:1.1:1775764877.028461:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7800. 00010000:00000010:2.1:1775764877.048260:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899600. 00000080:00000010:3.1:1775764877.048371:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6aa40. 00010000:00000010:2.1:1775764877.060218:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1600. 00010000:00000010:3.1:1775764877.060277:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6a00. 00000080:00000010:2.1:1775764877.069260:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b9c0. 00010000:00000010:0.1:1775764877.088276:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903a00. 00010000:00000010:3.1:1775764877.096020:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1a00. 00000080:00000010:3.1:1775764877.096045:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6abc0. 00010000:00000010:2.1:1775764877.111198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902400. 00010000:00000010:2.1:1775764877.126414:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00010000:00000010:2.1:1775764877.126421:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903400. 00000080:00000010:0.1:1775764877.175292:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8520. 00010000:00000010:0.1:1775764877.175307:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6c00. 00000080:00000010:2.1:1775764877.191233:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c83a0. 00010000:00000010:2.1:1775764877.191272:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d0ba00. 00010000:00000010:2.1:1775764877.191279:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902400. 00000080:00000010:2.1:1775764877.191286:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2840. 00000080:00000010:2.1:1775764877.191301:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275b8840. 00010000:00000010:3.1:1775764877.191823:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7c00. 00000080:00000010:0.1:1775764877.215195:0:2392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6aa00. 00010000:00000010:3.1:1775764877.215735:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1400. 00010000:00000010:2.1:1775764877.224223:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902600. 00010000:00000010:3.1:1775764877.244312:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7800. 00010000:00000010:2.1:1775764877.256181:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7400. 00000080:00000010:3.1:1775764877.274188:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2ca0. 00000080:00000010:2.1:1775764877.277000:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2520. 00010000:00000010:2.1:1775764877.277023:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207926200. 00010000:00000010:2.1:1775764877.277030:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898400. 00000080:00000010:2.1:1775764877.288363:0:2402:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b160. 00000080:00000010:2.1:1775764877.288397:0:2402:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d98c0. 00010000:00000010:2.1:1775764877.299009:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898c00. 00010000:00000010:0.1:1775764877.307564:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899200. 00000080:00000010:0.1:1775764877.307576:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b800. 00010000:00000010:2.1:1775764877.328200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902600. 00010000:00000010:3.1:1775764877.347373:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7400. 00010000:00000010:2.1:1775764877.354391:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1600. 00010000:00000010:2.1:1775764877.354446:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6e00. 00010000:00000010:2.1:1775764877.354450:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6800. 00000080:00000010:2.1:1775764877.367985:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a900. 00000080:00000010:2.1:1775764877.368003:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6be60. 00000080:00000010:2.1:1775764877.383347:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2520. 00010000:00000010:2.1:1775764877.396756:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902600. 00000080:00000010:2.1:1775764877.396769:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2840. 00010000:00000010:2.1:1775764877.400513:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902400. 00010000:00000010:1.1:1775764877.401218:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6c00. 00000080:00000010:1.1:1775764877.401229:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8e80. 00010000:00000010:0.1:1775764877.447187:0:6159:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420550be00. 00010000:00000010:0.1:1775764877.447195:0:6159:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a6600. 00010000:00000010:0.1:1775764877.447198:0:6159:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078e9800. 00010000:00000010:0.1:1775764877.447203:0:6159:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994201482600. 00010000:00000010:3.1:1775764877.481745:0:2402:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209d57300. 00010000:00000010:3.1:1775764877.481755:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793c600. 00010000:00000010:3.1:1775764877.481759:0:2402:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078b3100. 00010000:00000010:3.1:1775764877.481764:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079daa00. 00010000:00000010:3.1:1775764877.481767:0:2402:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420794c000. 00010000:00000010:3.1:1775764877.481770:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789d600. 00010000:00000010:1.1:1775764877.490589:0:6168:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078b3500. 00010000:00000010:1.1:1775764877.490615:0:6168:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789ce00. 00010000:00000010:3.1:1775764877.491807:0:6160:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207c67200. 00010000:00000010:3.1:1775764877.491816:0:6160:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789d800. 00010000:00000010:3.1:1775764877.491820:0:6160:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078a1300. 00010000:00000010:3.1:1775764877.491824:0:6160:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dae00. 00010000:00000010:3.1:1775764877.491828:0:6160:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209d57000. 00010000:00000010:3.1:1775764877.491833:0:6160:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a7800. 00010000:00000010:3.1:1775764877.491836:0:6160:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078a1400. 00010000:00000010:3.1:1775764877.491839:0:6160:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894200. 00010000:00000010:2.1:1775764877.498606:0:14473:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902400. 00010000:00000010:1.1:1775764877.498699:0:2401:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942420d9c00. 00010000:00000010:1.1:1775764877.498706:0:2401:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994200030800. 00010000:00000010:3.1:1775764877.499775:0:14464:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209d57b00. 00010000:00000010:3.1:1775764877.499782:0:14464:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cbd800. 00010000:00000010:3.1:1775764877.499786:0:14464:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207ce3900. 00010000:00000010:3.1:1775764877.499789:0:14464:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fde00. 00010000:00000010:3.1:1775764877.499793:0:14464:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209d57200. 00010000:00000010:3.1:1775764877.499796:0:14464:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0a00. 00010000:00000010:3.1:1775764877.499799:0:14464:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078a1600. 00010000:00000010:3.1:1775764877.499802:0:14464:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899000. 00010000:00000010:3.1:1775764877.499806:0:14464:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ef000. 00010000:00000010:3.1:1775764877.499809:0:14464:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793ca00. 00010000:00000010:3.1:1775764877.499815:0:14464:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420b307e00. 00010000:00000010:3.1:1775764877.499818:0:14464:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898000. 00010000:00000010:2.1:1775764877.503788:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902600. 00010000:00000010:2.1:1775764877.503944:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1200. 00010000:00000010:2.1:1775764877.503949:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1c00. 00010000:00000010:2.1:1775764877.503985:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903600. 00010000:00000010:1.1:1775764877.504436:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078efb00. 00010000:00000010:1.1:1775764877.504443:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da800. 00010000:00000010:1.1:1775764877.504447:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994205348b00. 00010000:00000010:1.1:1775764877.504450:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c200. 00010000:00000010:2.1:1775764877.514874:0:6169:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942069a5f00. 00010000:00000010:2.1:1775764877.514889:0:6169:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8600. 00010000:00000010:2.1:1775764877.514895:0:6169:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078b3b00. 00010000:00000010:2.1:1775764877.514899:0:6169:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fce00. 00010000:00000010:2.1:1775764877.514904:0:6169:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078e9900. 00010000:00000010:2.1:1775764877.514912:0:6169:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd200. 00010000:00000010:2.1:1775764877.514917:0:6169:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078a1f00. 00010000:00000010:2.1:1775764877.514921:0:6169:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899c00. 00010000:00000010:2.1:1775764877.514924:0:6169:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078a1200. 00010000:00000010:2.1:1775764877.514928:0:6169:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903800. 00010000:00000010:2.1:1775764877.514938:0:6169:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942420d9700. 00010000:00000010:2.1:1775764877.514942:0:6169:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902800. 00010000:00000010:3.1:1775764877.515993:0:14464:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078e9a00. 00010000:00000010:3.1:1775764877.516003:0:14464:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898800. 00010000:00000010:1.1:1775764877.517207:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207ce3200. 00010000:00000010:1.1:1775764877.517217:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994218f6ee00. 00010000:00000010:2.1:1775764877.521785:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942420d9900. 00010000:00000010:2.1:1775764877.521803:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994201483600. 00010000:00000010:2.1:1775764877.526336:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207ce3100. 00010000:00000010:2.1:1775764877.526347:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793aa00. 00010000:00000010:2.1:1775764877.526352:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994220010b00. 00010000:00000010:2.1:1775764877.526355:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793c000. 00010000:00000010:1.1:1775764877.527208:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209897f00. 00010000:00000010:1.1:1775764877.527217:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079db800. 00000080:00000010:2.1:1775764877.541194:0:2403:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2520. 00000080:00000010:2.1:1775764877.541209:0:2403:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2660. 00010000:00000010:2.1:1775764877.560180:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902800. 00000080:00000010:2.1:1775764877.560192:0:2402:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2a20. 00010000:00000010:2.1:1775764877.567264:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078e9200. 00010000:00000010:2.1:1775764877.567272:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d89400. 00010000:00000010:2.1:1775764877.567276:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209897200. 00010000:00000010:2.1:1775764877.567279:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793be00. 00010000:00000010:2.1:1775764877.567283:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209d57600. 00010000:00000010:2.1:1775764877.567286:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898600. 00010000:00000010:2.1:1775764877.584199:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942079a4800. 00010000:00000010:2.1:1775764877.584208:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899800. 00010000:00000010:2.1:1775764877.584214:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994210404000. 00010000:00000010:2.1:1775764877.584217:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079dbc00. 00010000:00000010:1.1:1775764877.584761:0:2392:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078a1100. 00010000:00000010:1.1:1775764877.584772:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207938200. 00010000:00000010:2.1:1775764877.594217:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994211cc1700. 00010000:00000010:2.1:1775764877.594235:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a6a00. 00010000:00000010:3.1:1775764877.612671:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6800. 00010000:00000010:3.1:1775764877.657060:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898800. 00010000:00000010:0.1:1775764877.657227:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903800. 00010000:00000010:2.1:1775764877.682899:0:2403:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943a00. 00010000:00000010:2.1:1775764877.682907:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902000. 00010000:00000010:2.1:1775764877.687299:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903600. 00010000:00000010:2.1:1775764877.698218:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902800. 00010000:00000010:3.1:1775764877.767389:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994206989300. 00010000:00000010:3.1:1775764877.767399:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7200. 00010000:00000010:2.1:1775764877.790967:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994218fe6200. 00010000:00000010:2.1:1775764877.790976:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d600. 00010000:00000010:2.1:1775764877.800049:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d0b800. 00010000:00000010:2.1:1775764877.800058:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d200. 00010000:00000010:2.1:1775764877.818987:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902800. 00010000:00000010:3.1:1775764877.963354:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903600. 00000080:00000010:2.1:1775764877.963749:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2a20. 00010000:00000010:2.1:1775764877.963762:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898800. 00010000:00000010:2.1:1775764878.001203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7400. 00000080:00000010:1.1:1775764878.046684:0:2392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2a20. 00010000:00000010:1.1:1775764878.046716:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7a00. 00010000:00000010:3.1:1775764878.062807:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902000. 00010000:00000010:2.1:1775764878.085137:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902800. 00000080:00000010:2.1:1775764878.106204:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2840. 00010000:00000010:2.1:1775764878.118212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902400. 00000080:00000010:3.1:1775764878.137295:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b7c0. 00010000:00000010:3.1:1775764878.152789:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902800. 00010000:00000010:2.1:1775764878.165464:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899000. 00010000:00000010:3.1:1775764878.183210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898800. 00010000:00000010:2.1:1775764878.196439:0:14473:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7200. 00000080:00000010:0.1:1775764878.211254:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2840. 00010000:00000010:0.1:1775764878.211269:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7600. 00010000:00000010:2.1:1775764878.219409:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902400. 00000080:00000010:0.1:1775764878.223204:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8620. 00010000:00000010:0.1:1775764878.223218:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7c00. 00000080:00000010:2.1:1775764878.240434:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2660. 00000080:00000010:0.1:1775764878.259185:0:14461:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b060. 00010000:00000010:3.1:1775764878.269278:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902400. 00010000:00000010:3.1:1775764878.287326:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7c00. 00010000:00000010:3.1:1775764878.304191:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902800. 00010000:00000010:2.1:1775764878.314187:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902000. 00010000:00000010:2.1:1775764878.377284:0:1358:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1e00. 00010000:00000010:2.1:1775764878.377293:0:1358:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903a00. 00010000:00000010:3.1:1775764878.389400:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902000. 00010000:00000010:3.1:1775764878.399433:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902200. 00000080:00000010:2.1:1775764878.406054:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a840. 00000080:00000010:3.1:1775764878.422980:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2820. 00010000:00000010:2.1:1775764878.427517:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7400. 00010000:00000010:2.1:1775764878.427524:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d0b700. 00010000:00000010:2.1:1775764878.427527:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903e00. 00010000:00000010:3.1:1775764878.429443:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903a00. 00010000:00000010:0.1:1775764878.449273:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898800. 00010000:00000010:0.1:1775764878.449279:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902a00. 00010000:00000010:2.1:1775764878.465655:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d0bc00. 00010000:00000010:2.1:1775764878.465671:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899800. 00000080:00000010:3.1:1775764878.470839:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b7c0. 00000080:00000010:3.1:1775764878.471209:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d8000. 00000080:00000010:0.1:1775764878.471247:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2fa0. 00000080:00000010:0.1:1775764878.471263:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275bd280. 00010000:00000010:0.1:1775764878.486445:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6800. 00000080:00000010:0.1:1775764878.486454:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8660. 00010000:00000010:2.1:1775764878.494990:0:1005:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7a00. 00000080:00000010:2.1:1775764878.495000:0:1005:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c81e0. 00010000:00000010:3.1:1775764878.529181:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903200. 00010000:00000010:2.1:1775764878.555234:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6200. 00010000:00000010:3.1:1775764878.563301:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7600. 00010000:00000010:3.1:1775764878.563322:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903c00. 00010000:00000010:3.1:1775764878.583198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789c400. 00010000:00000010:2.1:1775764878.590170:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789ca00. 00010000:00000010:3.1:1775764878.610432:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902e00. 00010000:00000010:3.1:1775764878.625568:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1a00. 00000080:00000010:3.1:1775764878.649228:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db25e0. 00000080:00000010:2.1:1775764878.653339:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8620. 00010000:00000010:1.1:1775764878.679189:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903200. 00010000:00000010:2.1:1775764878.679376:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898600. 00010000:00000010:2.1:1775764878.684201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7a00. 00010000:00000010:1.1:1775764878.705656:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903c00. 00010000:00000010:3.1:1775764878.729801:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902a00. 00010000:00000010:3.1:1775764878.745056:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899c00. 00000080:00000010:0.1:1775764878.756936:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db25e0. 00010000:00000010:0.1:1775764878.756949:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6400. 00010000:00000010:1.1:1775764878.764191:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902200. 00000080:00000010:2.1:1775764878.776186:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2f00. 00010000:00000010:2.1:1775764878.776201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902000. 00010000:00000010:2.1:1775764878.776206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903a00. 00000080:00000010:2.1:1775764878.794565:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bb20. 00010000:00000010:3.1:1775764878.805928:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903a00. 00000080:00000010:2.1:1775764878.810842:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db21e0. 00010000:00000010:3.1:1775764878.820896:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902000. 00010000:00000010:0.1:1775764878.840400:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902000. 00000080:00000010:2.1:1775764878.840464:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db21e0. 00010000:00000010:2.1:1775764878.840484:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902400. 00010000:00000010:3.1:1775764878.844196:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902800. 00000080:00000010:2.1:1775764878.869526:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2f00. 00010000:00000010:3.1:1775764878.888880:0:6168:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7600. 00010000:00000010:2.1:1775764878.915190:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902400. 00000080:00000010:0.1:1775764878.922725:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6aa00. 00010000:00000010:0.1:1775764878.922745:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1000. 00010000:00000010:0.1:1775764878.922748:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6800. 00010000:00000010:3.1:1775764878.939457:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903a00. 00000080:00000010:2.1:1775764878.942836:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2140. 00010000:00000010:3.1:1775764878.953203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902400. 00010000:00000010:2.1:1775764878.990610:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902200. 00010000:00000010:2.1:1775764879.043349:0:1026:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6400. 00010000:00000010:2.1:1775764879.043427:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903e00. 00010000:00000010:3.1:1775764879.047227:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0e00. 00010000:00000010:3.1:1775764879.047259:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7c00. 00000080:00000010:3.1:1775764879.100231:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c83c0. 00010000:00000010:3.1:1775764879.100245:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d0bc00. 00010000:00000010:3.1:1775764879.100249:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902200. 00000080:00000010:3.1:1775764879.100254:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2520. 00010000:00000010:3.1:1775764879.105130:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994206989300. 00010000:00000010:3.1:1775764879.105171:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899200. 00000080:00000010:3.1:1775764879.105179:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8980. 00000080:00000010:3.1:1775764879.105196:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227628840. 00000080:00000010:3.1:1775764879.121364:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a840. 00000080:00000010:3.1:1775764879.121393:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d8000. 00010000:00000010:0.1:1775764879.122203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903c00. 00000080:00000010:0.1:1775764879.126203:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2780. 00010000:00000010:3.1:1775764879.141316:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899200. 00000080:00000010:3.1:1775764879.146198:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bdc0. 00010000:00000010:2.1:1775764879.188334:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903200. 00010000:00000010:2.1:1775764879.188382:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6a00. 00010000:00000010:3.1:1775764879.199219:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899200. 00010000:00000010:3.1:1775764879.203541:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898400. 00010000:00000010:3.1:1775764879.234015:0:14473:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903200. 00010000:00000010:3.1:1775764879.261663:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898400. 00010000:00000010:2.1:1775764879.318213:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994206989300. 00010000:00000010:2.1:1775764879.318222:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00010000:00000010:2.1:1775764879.355566:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d0b600. 00010000:00000010:2.1:1775764879.355572:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902e00. 00010000:00000010:3.1:1775764879.363245:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898400. 00010000:00000010:3.1:1775764879.386395:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6200. 00010000:00000010:0.1:1775764879.459330:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d0b100. 00010000:00000010:0.1:1775764879.459342:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903400. 00010000:00000010:0.1:1775764879.459347:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d0b500. 00010000:00000010:0.1:1775764879.459367:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902c00. 00010000:00000010:0.1:1775764879.459372:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207de4600. 00010000:00000010:0.1:1775764879.459375:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6000. 00010000:00000010:3.1:1775764879.481210:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99421263ae00. 00010000:00000010:3.1:1775764879.481228:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899400. 00010000:00000010:1.1:1775764879.510083:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902e00. 00000080:00000010:2.1:1775764879.513212:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2820. 00010000:00000010:2.1:1775764879.513234:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6e00. 00000080:00000010:2.1:1775764879.595419:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2820. 00010000:00000010:2.1:1775764879.595437:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0a00. 00010000:00000010:3.1:1775764879.611731:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902400. 00010000:00000010:3.1:1775764879.631058:0:14473:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7c00. 00010000:00000010:1.1:1775764879.651957:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0000. 00000080:00000010:0.1:1775764879.670526:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bdc0. 00010000:00000010:2.1:1775764879.699437:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6c00. 00000080:00000010:2.1:1775764879.723384:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db21e0. 00010000:00000010:3.1:1775764879.729187:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7200. 00000080:00000010:1.1:1775764879.796430:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8520. 00010000:00000010:3.1:1775764879.801945:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902800. 00010000:00000010:3.1:1775764879.801953:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1a00. 00010000:00000010:3.1:1775764879.807262:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6c00. 00010000:00000010:3.1:1775764879.832865:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6200. 00010000:00000010:0.1:1775764879.832903:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902000. 00000080:00000010:3.1:1775764879.857724:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2820. 00000080:00000010:0.1:1775764879.867009:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6aac0. 00010000:00000010:0.1:1775764879.884200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6e00. 00010000:00000010:3.1:1775764879.900547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6400. 00010000:00000010:3.1:1775764879.900555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0a00. 00010000:00000010:0.1:1775764879.918533:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902400. 00010000:00000010:3.1:1775764879.930291:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899400. 00010000:00000010:3.1:1775764879.930316:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902e00. 00000080:00000010:2.1:1775764879.942470:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8000. 00010000:00000010:3.1:1775764879.970793:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903200. 00010000:00000010:3.1:1775764879.979130:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902c00. 00000080:00000010:2.1:1775764879.997554:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b300. 00010000:00000010:3.1:1775764880.017755:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902400. 00010000:00000010:3.1:1775764880.025766:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902e00. 00010000:00000010:3.1:1775764880.025774:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903400. 00010000:00000010:2.1:1775764880.081522:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902800. 00000080:00000010:2.1:1775764880.081533:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b880. 00010000:00000010:2.1:1775764880.081545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902000. 00010000:00000010:1.1:1775764880.082334:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899400. 00000080:00000010:2.1:1775764880.103218:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2880. 00010000:00000010:3.1:1775764880.125472:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0400. 00000080:00000010:3.1:1775764880.136408:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6afc0. 00010000:00000010:0.1:1775764880.142404:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902000. 00010000:00000010:2.1:1775764880.150889:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6800. 00010000:00000010:2.1:1775764880.163235:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898400. 00010000:00000010:3.1:1775764880.180351:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902800. 00000080:00000010:3.1:1775764880.195822:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2a20. 00000080:00000010:3.1:1775764880.210289:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2f00. 00010000:00000010:3.1:1775764880.220591:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943700. 00010000:00000010:3.1:1775764880.220598:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902000. 00010000:00000010:0.1:1775764880.221191:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899200. 00000080:00000010:2.1:1775764880.221259:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b320. 00010000:00000010:2.1:1775764880.221271:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903400. 00000080:00000010:3.1:1775764880.253651:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2880. 00000080:00000010:3.1:1775764880.253667:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227606b40. 00000080:00000010:1.1:1775764880.265287:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8980. 00010000:00000010:3.1:1775764880.265741:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7200. 00010000:00000010:3.1:1775764880.301207:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1c00. 00010000:00000010:2.1:1775764880.301391:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899600. 00010000:00000010:3.1:1775764880.326702:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1400. 00000080:00000010:2.1:1775764880.347578:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6ba40. 00010000:00000010:3.1:1775764880.359201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902e00. 00000080:00000010:3.1:1775764880.359218:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db24e0. 00000080:00000010:0.1:1775764880.367805:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2fa0. 00010000:00000010:0.1:1775764880.375216:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966600. 00010000:00000010:0.1:1775764880.375223:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899a00. 00010000:00000010:2.1:1775764880.375258:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902400. 00000080:00000010:2.1:1775764880.375265:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2480. 00000080:00000010:0.1:1775764880.390202:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b020. 00000080:00000010:0.1:1775764880.411792:0:2392:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d98c0. 00010000:00000010:3.1:1775764880.436315:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899a00. 00010000:00000010:3.1:1775764880.436436:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903400. 00010000:00000010:3.1:1775764880.436441:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902c00. 00010000:00000010:3.1:1775764880.445205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6400. 00000080:00000010:3.1:1775764880.460219:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b380. 00000080:00000010:3.1:1775764880.460230:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2500. 00010000:00000010:3.1:1775764880.460239:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899600. 00000080:00000010:3.1:1775764880.460243:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b3e0. 00010000:00000010:2.1:1775764880.462321:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902400. 00000080:00000010:2.1:1775764880.504620:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8860. 00010000:00000010:0.1:1775764880.506377:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902400. 00010000:00000010:3.1:1775764880.520707:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899600. 00010000:00000010:2.1:1775764880.524276:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899a00. 00010000:00000010:3.1:1775764880.534198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903200. 00000080:00000010:3.1:1775764880.534209:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2e80. 00010000:00000010:2.1:1775764880.568601:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903400. 00010000:00000010:2.1:1775764880.576968:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899200. 00010000:00000010:2.1:1775764880.576980:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898400. 00010000:00000010:3.1:1775764880.577432:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902c00. 00000080:00000010:2.1:1775764880.602210:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db22a0. 00000080:00000010:2.1:1775764880.602226:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2480. 00010000:00000010:2.1:1775764880.625229:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903400. 00000080:00000010:2.1:1775764880.625239:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2120. 00010000:00000010:3.1:1775764880.664669:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d0b600. 00010000:00000010:3.1:1775764880.664681:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902400. 00010000:00000010:3.1:1775764880.669206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899600. 00010000:00000010:1.1:1775764880.669394:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902e00. 00010000:00000010:3.1:1775764880.697241:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0600. 00010000:00000010:3.1:1775764880.713257:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6e00. 00010000:00000010:3.1:1775764880.754178:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207de4600. 00010000:00000010:3.1:1775764880.754200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1a00. 00010000:00000010:0.1:1775764880.818816:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902000. 00010000:00000010:2.1:1775764880.819201:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d0b700. 00010000:00000010:2.1:1775764880.819208:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903e00. 00010000:00000010:2.1:1775764880.819213:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994206989500. 00010000:00000010:2.1:1775764880.819216:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898c00. 00010000:00000010:3.1:1775764880.819642:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943200. 00010000:00000010:3.1:1775764880.819648:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0000. 00000080:00000010:2.1:1775764880.856252:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a880. 00010000:00000010:2.1:1775764880.860265:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899600. 00010000:00000010:0.1:1775764880.861205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0200. 00010000:00000010:3.1:1775764880.898724:0:1358:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6c00. 00000080:00000010:3.1:1775764880.948345:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b3e0. 00000080:00000010:2.1:1775764880.954367:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6afc0. 00010000:00000010:1.1:1775764880.960180:0:6159:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902800. 00010000:00000010:3.1:1775764880.960195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0400. 00010000:00000010:2.1:1775764880.974221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00010000:00000010:2.1:1775764880.990546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6400. 00010000:00000010:2.1:1775764880.997198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6800. 00000080:00000010:1.1:1775764881.016358:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a660. 00010000:00000010:1.1:1775764881.016378:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1e00. 00010000:00000010:0.1:1775764881.024197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0e00. 00000080:00000010:0.1:1775764881.044928:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b4a0. 00000080:00000010:2.1:1775764881.058204:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b380. 00010000:00000010:3.1:1775764881.066201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7c00. 00010000:00000010:3.1:1775764881.081195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903e00. 00010000:00000010:2.1:1775764881.095258:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7200. 00010000:00000010:0.1:1775764881.103890:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902800. 00010000:00000010:3.1:1775764881.123199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899800. 00010000:00000010:3.1:1775764881.139791:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902e00. 00010000:00000010:1.1:1775764881.139842:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902400. 00000080:00000010:2.1:1775764881.140187:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2480. 00010000:00000010:2.1:1775764881.140198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6c00. 00000080:00000010:2.1:1775764881.164201:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db28a0. 00000080:00000010:2.1:1775764881.172223:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b3e0. 00010000:00000010:0.1:1775764881.196187:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903200. 00010000:00000010:2.1:1775764881.196370:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902c00. 00010000:00000010:3.1:1775764881.203177:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6000. 00010000:00000010:3.1:1775764881.203184:0:2402:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d0bc00. 00010000:00000010:3.1:1775764881.203187:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902200. 00000080:00000010:3.1:1775764881.226350:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db22a0. 00000080:00000010:3.1:1775764881.226369:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d8000. 00010000:00000010:3.1:1775764881.231827:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902400. 00010000:00000010:2.1:1775764881.252270:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902c00. 00000080:00000010:2.1:1775764881.252278:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2900. 00010000:00000010:2.1:1775764881.260230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0600. 00010000:00000010:3.1:1775764881.297135:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0400. 00010000:00000010:2.1:1775764881.303237:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903e00. 00000080:00000010:2.1:1775764881.319305:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c81e0. 00010000:00000010:2.1:1775764881.329417:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0e00. 00010000:00000010:2.1:1775764881.353348:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902c00. 00010000:00000010:2.1:1775764881.353356:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943100. 00010000:00000010:2.1:1775764881.353360:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903a00. 00010000:00000010:3.1:1775764881.356651:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903e00. 00010000:00000010:2.1:1775764881.370851:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7400. 00000080:00000010:2.1:1775764881.374176:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2900. 00000080:00000010:2.1:1775764881.389142:0:2402:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275b8840. 00010000:00000010:3.1:1775764881.424376:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420788e500. 00010000:00000010:3.1:1775764881.424382:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994209969000. 00010000:00000010:0.1:1775764881.428390:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7c00. 00000080:00000010:0.1:1775764881.428397:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c82e0. 00010000:00000010:2.1:1775764881.429429:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903a00. 00000080:00000010:3.1:1775764881.449228:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8980. 00010000:00000010:0.1:1775764881.451181:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898800. 00010000:00000010:3.1:1775764881.470263:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899000. 00010000:00000010:2.1:1775764881.479927:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902000. 00010000:00000010:2.1:1775764881.479940:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902c00. 00010000:00000010:1.1:1775764881.501192:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903e00. 00010000:00000010:0.1:1775764881.538924:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903a00. 00010000:00000010:2.1:1775764881.556214:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420788e000. 00010000:00000010:2.1:1775764881.556221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994209968400. 00000080:00000010:3.1:1775764881.570391:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b3a0. 00010000:00000010:2.1:1775764881.572642:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899000. 00010000:00000010:0.1:1775764881.578475:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898800. 00010000:00000010:2.1:1775764881.597238:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1c00. 00000080:00000010:0.1:1775764881.608208:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2480. 00010000:00000010:2.1:1775764881.614229:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6e00. 00010000:00000010:2.1:1775764881.614234:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00010000:00000010:2.1:1775764881.621492:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902e00. 00000080:00000010:0.1:1775764881.627199:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b6a0. 00010000:00000010:2.1:1775764881.641936:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6400. 00010000:00000010:3.1:1775764881.645189:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899600. 00010000:00000010:2.1:1775764881.654060:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902c00. 00010000:00000010:2.1:1775764881.666190:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7600. 00000080:00000010:0.1:1775764881.670232:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8640. 00010000:00000010:2.1:1775764881.680065:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7200. 00000080:00000010:0.1:1775764881.683506:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8020. 00010000:00000010:0.1:1775764881.683522:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7400. 00010000:00000010:3.1:1775764881.688543:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902e00. 00010000:00000010:2.1:1775764881.690881:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6400. 00010000:00000010:2.1:1775764881.704195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7c00. 00010000:00000010:0.1:1775764881.717427:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6400. 00000080:00000010:0.1:1775764881.717496:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2240. 00010000:00000010:0.1:1775764881.717501:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902800. 00000080:00000010:0.1:1775764881.728458:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c83c0. 00010000:00000010:1.1:1775764881.745091:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6400. 00000080:00000010:2.1:1775764881.751604:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6abc0. 00010000:00000010:2.1:1775764881.764741:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899600. 00000080:00000010:2.1:1775764881.769485:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8720. 00010000:00000010:2.1:1775764881.769511:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207de4d00. 00010000:00000010:2.1:1775764881.769517:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7400. 00010000:00000010:3.1:1775764881.769565:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7200. 00000080:00000010:3.1:1775764881.804393:0:14464:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8320. 00000080:00000010:3.1:1775764881.806542:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff99422762b180. 00000080:00000010:2.1:1775764881.811727:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2d60. 00010000:00000010:2.1:1775764881.811742:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6200. 00010000:00000010:3.1:1775764881.812199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902c00. 00010000:00000010:2.1:1775764881.843190:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7800. 00010000:00000010:0.1:1775764881.856198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0400. 00000080:00000010:0.1:1775764881.856243:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bc60. 00010000:00000010:0.1:1775764881.856251:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6c00. 00010000:00000010:2.1:1775764881.881290:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1a00. 00000080:00000010:0.1:1775764881.900544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2900. 00000080:00000010:0.1:1775764881.909190:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8920. 00000080:00000010:2.1:1775764881.917202:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b820. 00010000:00000010:1.1:1775764881.921098:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d0b700. 00010000:00000010:1.1:1775764881.921106:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7200. 00010000:00000010:2.1:1775764881.924231:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902200. 00000080:00000010:1.1:1775764881.929727:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a080. 00010000:00000010:2.1:1775764881.940205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903200. 00000080:00000010:2.1:1775764881.940222:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db20a0. 00000080:00000010:0.1:1775764881.944468:0:14453:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275b8840. 00010000:00000010:3.1:1775764881.962289:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7400. 00010000:00000010:2.1:1775764881.967744:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903200. 00010000:00000010:2.1:1775764881.967784:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902000. 00010000:00000010:2.1:1775764881.967789:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902c00. 00000080:00000010:2.1:1775764881.992697:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db20a0. 00000080:00000010:2.1:1775764881.992714:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2d60. 00010000:00000010:2.1:1775764882.004927:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7800. 00010000:00000010:2.1:1775764882.004935:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902c00. 00000080:00000010:2.1:1775764882.004940:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2b80. 00000080:00000010:1.1:1775764882.028301:0:2392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8980. 00010000:00000010:1.1:1775764882.058219:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0a00. 00010000:00000010:2.1:1775764882.058408:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902c00. 00010000:00000010:2.1:1775764882.064179:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0000. 00010000:00000010:2.1:1775764882.073245:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6200. 00000080:00000010:2.1:1775764882.073258:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8720. 00010000:00000010:3.1:1775764882.090859:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7600. 00010000:00000010:2.1:1775764882.115628:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902200. 00010000:00000010:2.1:1775764882.115667:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1e00. 00010000:00000010:2.1:1775764882.115671:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903200. 00000080:00000010:1.1:1775764882.132796:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2d60. 00000080:00000010:1.1:1775764882.132815:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a5c0. 00010000:00000010:1.1:1775764882.147615:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0400. 00000080:00000010:1.1:1775764882.147629:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a580. 00010000:00000010:3.1:1775764882.151366:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966a00. 00010000:00000010:3.1:1775764882.151383:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902000. 00010000:00000010:2.1:1775764882.169236:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898c00. 00010000:00000010:1.1:1775764882.188869:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0400. 00010000:00000010:0.1:1775764882.193184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1800. 00010000:00000010:1.1:1775764882.236212:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d0bf00. 00010000:00000010:1.1:1775764882.236224:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6a00. 00010000:00000010:2.1:1775764882.239328:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898400. 00010000:00000010:2.1:1775764882.277149:0:1358:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966900. 00010000:00000010:2.1:1775764882.277174:0:1358:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899e00. 00010000:00000010:2.1:1775764882.277178:0:1358:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966200. 00010000:00000010:2.1:1775764882.277181:0:1358:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903c00. 00000080:00000010:0.1:1775764882.313213:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a880. 00010000:00000010:0.1:1775764882.313240:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0600. 00010000:00000010:3.1:1775764882.316787:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1000. 00010000:00000010:2.1:1775764882.333253:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966a00. 00010000:00000010:2.1:1775764882.333262:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7200. 00010000:00000010:3.1:1775764882.401296:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903200. 00010000:00000010:0.1:1775764882.441305:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899200. 00000080:00000010:0.1:1775764882.502296:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2b80. 00010000:00000010:0.1:1775764882.539488:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7400. 00010000:00000010:3.1:1775764882.601212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1e00. 00010000:00000010:3.1:1775764882.636792:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903a00. 00000080:00000010:2.1:1775764882.636871:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2640. 00010000:00000010:2.1:1775764882.636883:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903c00. 00000080:00000010:1.1:1775764882.637181:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a760. 00010000:00000010:2.1:1775764882.640699:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1a00. 00010000:00000010:1.1:1775764882.641026:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902e00. 00000080:00000010:2.1:1775764882.681214:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8e80. 00010000:00000010:3.1:1775764882.712339:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902800. 00010000:00000010:3.1:1775764882.736219:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903c00. 00010000:00000010:3.1:1775764882.766441:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903e00. 00000080:00000010:2.1:1775764882.789239:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a580. 00010000:00000010:3.1:1775764882.802841:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7200. 00010000:00000010:3.1:1775764882.814413:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903e00. 00010000:00000010:2.1:1775764882.829203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1200. 00000080:00000010:1.1:1775764882.844445:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8300. 00010000:00000010:3.1:1775764882.860417:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903c00. 00010000:00000010:3.1:1775764882.860428:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0000. 00010000:00000010:2.1:1775764882.866491:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902800. 00010000:00000010:2.1:1775764882.866515:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966500. 00010000:00000010:2.1:1775764882.866521:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899400. 00000080:00000010:2.1:1775764882.883644:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6afc0. 00000080:00000010:2.1:1775764882.892356:0:14453:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275beb40. 00010000:00000010:3.1:1775764882.896278:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1000. 00000080:00000010:2.1:1775764882.900311:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a260. 00010000:00000010:2.1:1775764882.900338:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902e00. 00010000:00000010:2.1:1775764882.927218:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7400. 00010000:00000010:2.1:1775764882.952382:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420b14a000. 00000080:00000010:2.1:1775764882.989767:0:1358:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db20a0. 00010000:00000010:3.1:1775764882.992965:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6200. 00010000:00000010:3.1:1775764883.019196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902e00. 00010000:00000010:2.1:1775764883.044193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899400. 00010000:00000010:1.1:1775764883.073872:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1800. 00010000:00000010:3.1:1775764883.080542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994209db3400. 00010000:00000010:3.1:1775764883.080555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899e00. 00010000:00000010:2.1:1775764883.114220:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6400. 00010000:00000010:2.1:1775764883.129797:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898400. 00000080:00000010:3.1:1775764883.144242:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bd00. 00010000:00000010:3.1:1775764883.157193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899e00. 00010000:00000010:0.1:1775764883.157230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903a00. 00010000:00000010:0.1:1775764883.157238:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966000. 00010000:00000010:0.1:1775764883.157242:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898600. 00000080:00000010:2.1:1775764883.164820:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227628840. 00000080:00000010:3.1:1775764883.166079:0:14464:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bd20. 00010000:00000010:3.1:1775764883.180500:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1800. 00010000:00000010:3.1:1775764883.180522:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903200. 00010000:00000010:2.1:1775764883.181634:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca1400. 00010000:00000010:2.1:1775764883.198287:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420b14ac00. 00010000:00000010:2.1:1775764883.198296:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0200. 00010000:00000010:2.1:1775764883.249406:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902000. 00010000:00000010:3.1:1775764883.257187:0:6169:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6a00. 00010000:00000010:3.1:1775764883.276565:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420b14b600. 00010000:00000010:2.1:1775764883.299326:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0a00. 00010000:00000010:2.1:1775764883.318414:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6e00. 00000080:00000010:2.1:1775764883.318421:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8980. 00010000:00000010:0.1:1775764883.377548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903a00. 00010000:00000010:3.1:1775764883.378076:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899e00. 00010000:00000010:3.1:1775764883.378088:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898c00. 00010000:00000010:2.1:1775764883.383817:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917200. 00010000:00000010:3.1:1775764883.424224:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898c00. 00010000:00000010:3.1:1775764883.424232:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902e00. 00010000:00000010:0.1:1775764883.514029:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903c00. 00010000:00000010:3.1:1775764883.528312:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898c00. 00000080:00000010:3.1:1775764883.530026:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b8e0. 00000080:00000010:3.1:1775764883.530039:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bd00. 00000080:00000010:3.1:1775764883.530042:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bd20. 00010000:00000010:3.1:1775764883.530047:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902000. 00010000:00000010:3.1:1775764883.544331:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00010000:00000010:0.1:1775764883.569292:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903e00. 00000080:00000010:0.1:1775764883.569311:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2f80. 00010000:00000010:3.1:1775764883.583179:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00010000:00000010:1.1:1775764883.592229:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0200. 00000080:00000010:1.1:1775764883.592244:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a260. 00010000:00000010:1.1:1775764883.597257:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898c00. 00010000:00000010:3.1:1775764883.620319:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00010000:00000010:1.1:1775764883.621019:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0200. 00010000:00000010:0.1:1775764883.624531:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1000. 00010000:00000010:2.1:1775764883.647847:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1a00. 00010000:00000010:3.1:1775764883.669490:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7800. 00000080:00000010:3.1:1775764883.669526:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a260. 00000080:00000010:3.1:1775764883.669533:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bd00. 00000080:00000010:3.1:1775764883.669538:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bd20. 00010000:00000010:3.1:1775764883.669541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7c00. 00000080:00000010:1.1:1775764883.718550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2100. 00000080:00000010:2.1:1775764883.746876:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8320. 00000080:00000010:2.1:1775764883.759231:0:13:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6aa40. 00000080:00000010:2.1:1775764883.765663:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a9a0. 00010000:00000010:2.1:1775764883.765688:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d0bf00. 00010000:00000010:2.1:1775764883.765693:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6200. 00010000:00000010:3.1:1775764883.767805:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898800. 00000080:00000010:3.1:1775764883.819276:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b8e0. 00000080:00000010:3.1:1775764883.819308:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275beb40. 00010000:00000010:3.1:1775764883.841478:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898800. 00000080:00000010:3.1:1775764883.841497:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bd00. 00010000:00000010:3.1:1775764883.888193:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898800. 00010000:00000010:3.1:1775764883.888200:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902000. 00000080:00000010:3.1:1775764883.888205:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db20a0. 00010000:00000010:2.1:1775764883.912215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7c00. 00000080:00000010:2.1:1775764883.912290:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bd00. 00010000:00000010:2.1:1775764883.912298:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898c00. 00010000:00000010:3.1:1775764883.950805:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1c00. 00010000:00000010:3.1:1775764883.956315:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898600. 00010000:00000010:1.1:1775764883.962069:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0000. 00000080:00000010:1.1:1775764883.967204:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a760. 00000080:00000010:3.1:1775764884.007977:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2320. 00010000:00000010:2.1:1775764884.022192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902e00. 00000080:00000010:3.1:1775764884.032897:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db26c0. 00000080:00000010:3.1:1775764884.066579:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8860. 00000080:00000010:1.1:1775764884.074191:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2680. 00010000:00000010:1.1:1775764884.074209:0:2393:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966a00. 00010000:00000010:1.1:1775764884.074216:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898800. 00000080:00000010:3.1:1775764884.086777:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a820. 00000080:00000010:2.1:1775764884.097278:0:14454:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275dca40. 00010000:00000010:1.1:1775764884.110561:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902e00. 00000080:00000010:1.1:1775764884.110571:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db21a0. 00010000:00000010:3.1:1775764884.129461:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d0bf00. 00010000:00000010:3.1:1775764884.129477:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903c00. 00010000:00000010:2.1:1775764884.137541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902000. 00010000:00000010:3.1:1775764884.181280:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898600. 00010000:00000010:0.1:1775764884.188206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903c00. 00000080:00000010:2.1:1775764884.188319:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2380. 00010000:00000010:2.1:1775764884.188329:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966d00. 00010000:00000010:2.1:1775764884.188332:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6c00. 00010000:00000010:2.1:1775764884.188336:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899400. 00000080:00000010:2.1:1775764884.188972:0:26:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b9a0. 00010000:00000010:2.1:1775764884.188982:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0600. 00010000:00000010:1.1:1775764884.246830:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1c00. 00000080:00000010:1.1:1775764884.246846:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a440. 00000080:00000010:0.1:1775764884.249141:0:2392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a9a0. 00010000:00000010:0.1:1775764884.249177:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1800. 00010000:00000010:3.1:1775764884.250573:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0000. 00010000:00000010:3.1:1775764884.289351:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7c00. 00000080:00000010:2.1:1775764884.319433:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8000. 00010000:00000010:2.1:1775764884.354242:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994203fb1600. 00010000:00000010:3.1:1775764884.364421:0:2402:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966300. 00010000:00000010:3.1:1775764884.364434:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902e00. 00010000:00000010:0.1:1775764884.382350:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994203fb0200. 00010000:00000010:3.1:1775764884.390893:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0200. 00000080:00000010:3.1:1775764884.406858:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2380. 00010000:00000010:0.1:1775764884.415212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898600. 00010000:00000010:3.1:1775764884.429311:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d0b100. 00010000:00000010:3.1:1775764884.429322:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994203fb0200. 00000080:00000010:2.1:1775764884.432849:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2cc0. 00010000:00000010:2.1:1775764884.432866:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994203fb1600. 00010000:00000010:3.1:1775764884.446247:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899400. 00010000:00000010:3.1:1775764884.446305:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1e00. 00010000:00000010:3.1:1775764884.446309:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0800. 00000080:00000010:0.1:1775764884.450214:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b5e0. 00000080:00000010:0.1:1775764884.450237:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b4e0. 00010000:00000010:3.1:1775764884.495336:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7400. 00010000:00000010:1.1:1775764884.553039:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1c00. 00010000:00000010:1.1:1775764884.553063:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994203fb0200. 00000080:00000010:2.1:1775764884.564194:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2cc0. 00010000:00000010:2.1:1775764884.564212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899400. 00000080:00000010:2.1:1775764884.614207:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b260. 00010000:00000010:3.1:1775764884.618260:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1c00. 00010000:00000010:0.1:1775764884.649190:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966700. 00010000:00000010:0.1:1775764884.649196:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899600. 00000080:00000010:0.1:1775764884.658645:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b6e0. 00000080:00000010:0.1:1775764884.658668:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275deb40. 00010000:00000010:2.1:1775764884.671393:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994203fb0200. 00010000:00000010:3.1:1775764884.678807:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6200. 00000080:00000010:3.1:1775764884.678817:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8e80. 00010000:00000010:2.1:1775764884.709229:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898800. 00000080:00000010:3.1:1775764884.753390:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2e60. 00010000:00000010:3.1:1775764884.765209:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6a00. 00010000:00000010:2.1:1775764884.778226:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7a00. 00010000:00000010:2.1:1775764884.787483:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994203fb0200. 00010000:00000010:2.1:1775764884.787494:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7800. 00010000:00000010:3.1:1775764884.797670:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0800. 00000080:00000010:3.1:1775764884.800448:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2460. 00000080:00000010:3.1:1775764884.800470:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bce0. 00010000:00000010:2.1:1775764884.835513:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7a00. 00010000:00000010:2.1:1775764884.835522:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898800. 00010000:00000010:2.1:1775764884.859206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1400. 00010000:00000010:2.1:1775764884.863226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7a00. 00000080:00000010:2.1:1775764884.863365:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a780. 00000080:00000010:2.1:1775764884.863373:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2940. 00010000:00000010:2.1:1775764884.863378:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898c00. 00010000:00000010:2.1:1775764884.885265:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899600. 00010000:00000010:3.1:1775764884.954420:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899400. 00010000:00000010:0.1:1775764884.958333:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7a00. 00010000:00000010:2.1:1775764884.997341:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966e00. 00010000:00000010:2.1:1775764884.997352:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00000080:00000010:2.1:1775764884.997357:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8980. 00000080:00000010:2.1:1775764884.997373:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275db180. 00010000:00000010:2.1:1775764885.007190:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6a00. 00000080:00000010:2.1:1775764885.007204:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2940. 00010000:00000010:0.1:1775764885.017401:0:2392:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966300. 00010000:00000010:0.1:1775764885.017502:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7c00. 00010000:00000010:3.1:1775764885.053205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6a00. 00010000:00000010:1.1:1775764885.072901:0:1358:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6a00. 00000080:00000010:1.1:1775764885.085397:0:2401:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8020. 00010000:00000010:1.1:1775764885.085761:0:2401:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7800. 00010000:00000010:1.1:1775764885.085768:0:2401:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207de4700. 00010000:00000010:1.1:1775764885.085772:0:2401:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7c00. 00010000:00000010:2.1:1775764885.100611:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6400. 00010000:00000010:2.1:1775764885.117646:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6e00. 00010000:00000010:2.1:1775764885.143210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6400. 00000080:00000010:1.1:1775764885.143331:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8e80. 00010000:00000010:3.1:1775764885.146193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7800. 00000080:00000010:0.1:1775764885.146359:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db27a0. 00000080:00000010:0.1:1775764885.146368:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2940. 00010000:00000010:0.1:1775764885.146371:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7e00. 00010000:00000010:1.1:1775764885.146970:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899400. 00010000:00000010:1.1:1775764885.193715:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898600. 00010000:00000010:1.1:1775764885.209131:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899400. 00000080:00000010:1.1:1775764885.209141:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bd80. 00000080:00000010:2.1:1775764885.259175:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c81e0. 00010000:00000010:3.1:1775764885.268081:0:2402:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d0b200. 00010000:00000010:3.1:1775764885.268092:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7400. 00010000:00000010:2.1:1775764885.285330:0:1358:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6e00. 00010000:00000010:3.1:1775764885.288687:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6400. 00010000:00000010:3.1:1775764885.318138:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6c00. 00010000:00000010:3.1:1775764885.331195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7000. 00000080:00000010:0.1:1775764885.336636:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db22c0. 00010000:00000010:0.1:1775764885.336664:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d0b100. 00010000:00000010:0.1:1775764885.336670:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7600. 00010000:00000010:3.1:1775764885.342851:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6e00. 00000080:00000010:2.1:1775764885.385514:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2b20. 00010000:00000010:3.1:1775764885.390301:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6600. 00010000:00000010:1.1:1775764885.423370:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7800. 00010000:00000010:1.1:1775764885.428526:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0200. 00010000:00000010:1.1:1775764885.428535:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0a00. 00000080:00000010:1.1:1775764885.439449:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c81e0. 00000080:00000010:1.1:1775764885.439472:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2e40. 00010000:00000010:0.1:1775764885.439719:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6600. 00000080:00000010:2.1:1775764885.446620:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2b20. 00010000:00000010:2.1:1775764885.446644:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00010000:00000010:3.1:1775764885.488561:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0000. 00010000:00000010:2.1:1775764885.493204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6e00. 00000080:00000010:2.1:1775764885.493211:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2b20. 00010000:00000010:3.1:1775764885.515209:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899600. 00000080:00000010:3.1:1775764885.553368:0:6168:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bec0. 00010000:00000010:3.1:1775764885.565189:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899600. 00010000:00000010:0.1:1775764885.574787:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6e00. 00000080:00000010:1.1:1775764885.576208:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2b20. 00010000:00000010:1.1:1775764885.586385:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898c00. 00010000:00000010:3.1:1775764885.603565:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1c00. 00000080:00000010:3.1:1775764885.623449:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2be0. 00010000:00000010:3.1:1775764885.628983:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7000. 00000080:00000010:2.1:1775764885.653200:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bf00. 00010000:00000010:3.1:1775764885.658104:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7e00. 00010000:00000010:2.1:1775764885.666206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6a00. 00010000:00000010:2.1:1775764885.666260:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6400. 00010000:00000010:2.1:1775764885.666264:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7600. 00010000:00000010:3.1:1775764885.669337:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899600. 00000080:00000010:2.1:1775764885.690180:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2b00. 00000080:00000010:2.1:1775764885.690194:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2220. 00010000:00000010:3.1:1775764885.724235:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7800. 00010000:00000010:2.1:1775764885.731978:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7600. 00010000:00000010:3.1:1775764885.769390:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899600. 00000080:00000010:0.1:1775764885.776299:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bec0. 00010000:00000010:3.1:1775764885.784182:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7800. 00010000:00000010:3.1:1775764885.796199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6400. 00000080:00000010:2.1:1775764885.803200:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2b00. 00010000:00000010:0.1:1775764885.803296:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6a00. 00000080:00000010:0.1:1775764885.803302:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c81e0. 00010000:00000010:2.1:1775764885.807198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7600. 00000080:00000010:3.1:1775764885.854285:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2300. 00010000:00000010:3.1:1775764885.878275:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0200. 00010000:00000010:3.1:1775764885.878310:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0000. 00010000:00000010:3.1:1775764885.878313:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7c00. 00000080:00000010:3.1:1775764885.882210:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a080. 00000080:00000010:3.1:1775764885.882226:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8320. 00010000:00000010:1.1:1775764885.907873:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6a00. 00010000:00000010:2.1:1775764885.927197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899600. 00010000:00000010:1.1:1775764885.951203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898800. 00010000:00000010:1.1:1775764885.963316:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7600. 00000080:00000010:2.1:1775764885.977728:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b400. 00010000:00000010:3.1:1775764885.979736:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6a00. 00010000:00000010:0.1:1775764885.989203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7400. 00000080:00000010:1.1:1775764885.989278:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8520. 00010000:00000010:1.1:1775764885.989291:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7a00. 00010000:00000010:3.1:1775764885.997199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6400. 00000080:00000010:3.1:1775764885.997206:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2440. 00010000:00000010:2.1:1775764886.033193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7c00. 00000080:00000010:2.1:1775764886.033216:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c83c0. 00010000:00000010:2.1:1775764886.053340:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6e00. 00010000:00000010:0.1:1775764886.059525:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7c00. 00010000:00000010:0.1:1775764886.059529:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7e00. 00010000:00000010:0.1:1775764886.068207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7c00. 00000080:00000010:0.1:1775764886.085233:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2960. 00000080:00000010:0.1:1775764886.085255:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2b00. 00010000:00000010:2.1:1775764886.094820:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7800. 00010000:00000010:0.1:1775764886.114201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7400. 00000080:00000010:0.1:1775764886.114246:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8620. 00010000:00000010:0.1:1775764886.114253:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7000. 00010000:00000010:3.1:1775764886.119282:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1c00. 00010000:00000010:1.1:1775764886.127464:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7c00. 00010000:00000010:2.1:1775764886.149314:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6e00. 00010000:00000010:3.1:1775764886.187603:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1e00. 00000080:00000010:1.1:1775764886.196402:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a820. 00010000:00000010:1.1:1775764886.196413:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7800. 00000080:00000010:1.1:1775764886.238322:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8640. 00010000:00000010:0.1:1775764886.242397:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7c00. 00010000:00000010:2.1:1775764886.260352:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6e00. 00000080:00000010:2.1:1775764886.260366:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2560. 00010000:00000010:2.1:1775764886.323301:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6e00. 00010000:00000010:1.1:1775764886.324016:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1000. 00010000:00000010:1.1:1775764886.342838:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7000. 00010000:00000010:3.1:1775764886.351195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7e00. 00010000:00000010:0.1:1775764886.351389:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7c00. 00000080:00000010:0.1:1775764886.351509:0:2392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2560. 00010000:00000010:0.1:1775764886.351516:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0000. 00010000:00000010:2.1:1775764886.368279:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994206989500. 00010000:00000010:2.1:1775764886.368286:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1400. 00000080:00000010:2.1:1775764886.372737:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a820. 00000080:00000010:2.1:1775764886.372761:0:14454:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275beb40. 00010000:00000010:2.1:1775764886.384209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6400. 00010000:00000010:2.1:1775764886.391355:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7a00. 00000080:00000010:2.1:1775764886.391362:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2620. 00010000:00000010:3.1:1775764886.429335:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898800. 00010000:00000010:1.1:1775764886.432280:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1c00. 00000080:00000010:0.1:1775764886.438206:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a080. 00010000:00000010:3.1:1775764886.438290:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7a00. 00000080:00000010:2.1:1775764886.441966:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6be40. 00010000:00000010:2.1:1775764886.441982:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6a00. 00010000:00000010:2.1:1775764886.480208:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6a00. 00010000:00000010:3.1:1775764886.481257:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7600. 00000080:00000010:3.1:1775764886.481266:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2280. 00010000:00000010:2.1:1775764886.483390:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6e00. 00010000:00000010:2.1:1775764886.508382:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994210404900. 00010000:00000010:2.1:1775764886.508407:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7a00. 00000080:00000010:2.1:1775764886.527280:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2ec0. 00010000:00000010:3.1:1775764886.540253:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898800. 00000080:00000010:2.1:1775764886.540321:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275beb40. 00010000:00000010:2.1:1775764886.543315:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7a00. 00000080:00000010:2.1:1775764886.543328:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2ee0. 00010000:00000010:2.1:1775764886.568277:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966300. 00010000:00000010:2.1:1775764886.568291:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6400. 00010000:00000010:3.1:1775764886.579363:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898800. 00010000:00000010:3.1:1775764886.603242:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898c00. 00000080:00000010:3.1:1775764886.603281:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bc40. 00010000:00000010:2.1:1775764886.603345:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7c00. 00010000:00000010:3.1:1775764886.607221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7400. 00010000:00000010:2.1:1775764886.623209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7000. 00000080:00000010:2.1:1775764886.623221:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2700. 00010000:00000010:2.1:1775764886.632195:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942069a5f00. 00010000:00000010:2.1:1775764886.632208:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7e00. 00000080:00000010:3.1:1775764886.638201:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2ec0. 00010000:00000010:3.1:1775764886.638222:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6400. 00010000:00000010:3.1:1775764886.689213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898800. 00000080:00000010:0.1:1775764886.731299:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2700. 00010000:00000010:0.1:1775764886.739191:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cbd800. 00010000:00000010:3.1:1775764886.739416:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7000. 00000080:00000010:2.1:1775764886.746826:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2620. 00010000:00000010:3.1:1775764886.765270:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99421263ac00. 00010000:00000010:3.1:1775764886.765281:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7a00. 00010000:00000010:2.1:1775764886.781206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0a00. 00010000:00000010:2.1:1775764886.781215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6400. 00010000:00000010:3.1:1775764886.802489:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1e00. 00000080:00000010:2.1:1775764886.804227:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a760. 00010000:00000010:2.1:1775764886.804249:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207de4700. 00010000:00000010:2.1:1775764886.804255:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cbd600. 00010000:00000010:3.1:1775764886.812358:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7600. 00000080:00000010:1.1:1775764886.830547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2ee0. 00010000:00000010:2.1:1775764886.849799:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1e00. 00010000:00000010:1.1:1775764886.852338:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1400. 00010000:00000010:0.1:1775764886.864277:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0800. 00010000:00000010:0.1:1775764886.864282:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899600. 00000080:00000010:0.1:1775764886.869536:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6aac0. 00000080:00000010:0.1:1775764886.869548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a660. 00000080:00000010:3.1:1775764886.876328:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2620. 00010000:00000010:3.1:1775764886.876343:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7600. 00010000:00000010:0.1:1775764886.876589:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7a00. 00010000:00000010:2.1:1775764886.893198:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1600. 00000080:00000010:2.1:1775764886.902310:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b620. 00010000:00000010:3.1:1775764886.915744:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1400. 00010000:00000010:3.1:1775764886.928624:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7000. 00010000:00000010:3.1:1775764886.950235:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7600. 00010000:00000010:3.1:1775764886.957360:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7a00. 00010000:00000010:2.1:1775764886.961202:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d0bf00. 00010000:00000010:2.1:1775764886.961208:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6a00. 00000080:00000010:2.1:1775764886.972307:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c84a0. 00000080:00000010:2.1:1775764886.977300:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275bf380. 00000080:00000010:2.1:1775764886.984868:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2720. 00010000:00000010:1.1:1775764886.991212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7600. 00010000:00000010:2.1:1775764886.993378:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7a00. 00000080:00000010:2.1:1775764886.993394:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2800. 00010000:00000010:3.1:1775764886.993455:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898800. 00010000:00000010:3.1:1775764886.993488:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7000. 00010000:00000010:3.1:1775764886.993491:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7c00. 00000080:00000010:2.1:1775764887.017211:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bc40. 00000080:00000010:2.1:1775764887.017225:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db23e0. 00010000:00000010:3.1:1775764887.037302:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898800. 00010000:00000010:3.1:1775764887.037372:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994203fb0200. 00010000:00000010:3.1:1775764887.052298:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894200. 00010000:00000010:3.1:1775764887.073329:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895400. 00010000:00000010:2.1:1775764887.082178:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894400. 00010000:00000010:1.1:1775764887.134212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894400. 00010000:00000010:2.1:1775764887.138200:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942069a5a00. 00010000:00000010:2.1:1775764887.138214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00010000:00000010:3.1:1775764887.138968:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994211cc1700. 00010000:00000010:3.1:1775764887.138974:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7c00. 00000080:00000010:2.1:1775764887.150455:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2160. 00000080:00000010:2.1:1775764887.155995:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275b8840. 00010000:00000010:2.1:1775764887.173190:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895800. 00000080:00000010:2.1:1775764887.173199:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2620. 00010000:00000010:3.1:1775764887.176522:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0200. 00010000:00000010:2.1:1775764887.251199:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1600. 00000080:00000010:1.1:1775764887.260501:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c84a0. 00010000:00000010:1.1:1775764887.260514:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943f00. 00010000:00000010:1.1:1775764887.260519:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0200. 00010000:00000010:2.1:1775764887.264208:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895800. 00010000:00000010:2.1:1775764887.264358:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895e00. 00010000:00000010:3.1:1775764887.297194:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207de4000. 00010000:00000010:3.1:1775764887.297202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6000. 00010000:00000010:2.1:1775764887.297243:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966000. 00010000:00000010:2.1:1775764887.297251:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899000. 00010000:00000010:2.1:1775764887.297255:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966400. 00010000:00000010:2.1:1775764887.297257:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0e00. 00010000:00000010:2.1:1775764887.302872:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966200. 00010000:00000010:2.1:1775764887.302878:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898400. 00010000:00000010:2.1:1775764887.321667:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994206989500. 00010000:00000010:2.1:1775764887.321686:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6a00. 00010000:00000010:3.1:1775764887.322477:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d0bd00. 00010000:00000010:3.1:1775764887.322485:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7a00. 00010000:00000010:0.1:1775764887.338294:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d0b800. 00010000:00000010:0.1:1775764887.338304:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902400. 00010000:00000010:0.1:1775764887.338312:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d0b900. 00010000:00000010:0.1:1775764887.338318:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903e00. 00010000:00000010:0.1:1775764887.338325:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207de4800. 00010000:00000010:0.1:1775764887.338330:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903400. 00010000:00000010:0.1:1775764887.338336:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994206989b00. 00010000:00000010:0.1:1775764887.338342:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6e00. 00010000:00000010:2.1:1775764887.389271:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895e00. 00000080:00000010:2.1:1775764887.389279:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6be40. 00010000:00000010:2.1:1775764887.389289:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1e00. 00010000:00000010:1.1:1775764887.389727:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898800. 00010000:00000010:2.1:1775764887.418466:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966200. 00010000:00000010:2.1:1775764887.418474:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894c00. 00010000:00000010:2.1:1775764887.423271:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895e00. 00000080:00000010:2.1:1775764887.423284:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2580. 00010000:00000010:2.1:1775764887.456490:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1800. 00000080:00000010:2.1:1775764887.472248:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8020. 00010000:00000010:0.1:1775764887.488645:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895e00. 00010000:00000010:1.1:1775764887.499279:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207de4800. 00010000:00000010:1.1:1775764887.499298:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899400. 00000080:00000010:3.1:1775764887.499495:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b280. 00010000:00000010:3.1:1775764887.499510:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1400. 00010000:00000010:2.1:1775764887.503426:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1000. 00010000:00000010:2.1:1775764887.532501:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894400. 00000080:00000010:1.1:1775764887.551903:0:2400:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2580. 00010000:00000010:0.1:1775764887.560629:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1e00. 00000080:00000010:3.1:1775764887.566221:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b280. 00010000:00000010:3.1:1775764887.566242:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895e00. 00010000:00000010:3.1:1775764887.566247:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894400. 00010000:00000010:3.1:1775764887.566277:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894200. 00010000:00000010:3.1:1775764887.566280:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894c00. 00010000:00000010:1.1:1775764887.566319:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898600. 00000080:00000010:3.1:1775764887.572219:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2fc0. 00000080:00000010:3.1:1775764887.572242:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a9a0. 00010000:00000010:3.1:1775764887.589190:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898800. 00000080:00000010:3.1:1775764887.589199:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b280. 00010000:00000010:2.1:1775764887.615483:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899400. 00010000:00000010:0.1:1775764887.620260:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894c00. 00000080:00000010:3.1:1775764887.635151:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2800. 00010000:00000010:0.1:1775764887.645517:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898800. 00010000:00000010:2.1:1775764887.661354:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894600. 00010000:00000010:0.1:1775764887.664919:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894200. 00000080:00000010:3.1:1775764887.664986:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2e00. 00010000:00000010:3.1:1775764887.664998:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1c00. 00010000:00000010:3.1:1775764887.709713:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899000. 00000080:00000010:2.1:1775764887.735345:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b620. 00000080:00000010:2.1:1775764887.744198:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b280. 00010000:00000010:3.1:1775764887.762237:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894600. 00010000:00000010:1.1:1775764887.762350:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894400. 00010000:00000010:2.1:1775764887.784048:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895e00. 00010000:00000010:2.1:1775764887.784084:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0200. 00010000:00000010:2.1:1775764887.784088:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898400. 00010000:00000010:3.1:1775764887.792372:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899000. 00000080:00000010:3.1:1775764887.792380:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2c20. 00000080:00000010:3.1:1775764887.792513:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2060. 00000080:00000010:3.1:1775764887.812581:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2540. 00010000:00000010:2.1:1775764887.819320:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895e00. 00000080:00000010:2.1:1775764887.819328:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2160. 00010000:00000010:1.1:1775764887.826016:0:14461:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899000. 00010000:00000010:3.1:1775764887.834188:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899600. 00010000:00000010:3.1:1775764887.848547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899600. 00010000:00000010:3.1:1775764887.848608:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00010000:00000010:3.1:1775764887.848613:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894400. 00010000:00000010:3.1:1775764887.853184:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1c00. 00000080:00000010:3.1:1775764887.870291:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bd80. 00000080:00000010:3.1:1775764887.870303:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bec0. 00000080:00000010:2.1:1775764887.879285:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2160. 00010000:00000010:3.1:1775764887.882442:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895e00. 00010000:00000010:2.1:1775764887.883271:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0800. 00010000:00000010:3.1:1775764887.911819:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00010000:00000010:3.1:1775764887.962036:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899600. 00000080:00000010:3.1:1775764888.004740:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2620. 00000080:00000010:1.1:1775764888.024334:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2160. 00010000:00000010:3.1:1775764888.026623:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899600. 00010000:00000010:2.1:1775764888.028373:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894600. 00010000:00000010:3.1:1775764888.075392:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899000. 00010000:00000010:2.1:1775764888.099677:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1000. 00010000:00000010:3.1:1775764888.104578:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899600. 00000080:00000010:0.1:1775764888.122208:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bd80. 00010000:00000010:1.1:1775764888.127318:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0a00. 00010000:00000010:0.1:1775764888.142685:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1000. 00000080:00000010:0.1:1775764888.142701:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8000. 00010000:00000010:2.1:1775764888.202212:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966900. 00010000:00000010:2.1:1775764888.202240:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894600. 00010000:00000010:3.1:1775764888.216197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899600. 00010000:00000010:1.1:1775764888.216402:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1600. 00010000:00000010:1.1:1775764888.216408:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1c00. 00000080:00000010:1.1:1775764888.234132:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b6e0. 00000080:00000010:1.1:1775764888.234192:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275beb40. 00000080:00000010:3.1:1775764888.242536:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b260. 00000080:00000010:3.1:1775764888.242564:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8920. 00010000:00000010:3.1:1775764888.249120:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0800. 00000080:00000010:1.1:1775764888.261132:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6aa40. 00010000:00000010:1.1:1775764888.261170:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00010000:00000010:3.1:1775764888.297249:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894600. 00010000:00000010:2.1:1775764888.338208:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894c00. 00000080:00000010:2.1:1775764888.338353:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2ec0. 00000080:00000010:0.1:1775764888.350257:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8360. 00010000:00000010:0.1:1775764888.350288:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1e00. 00010000:00000010:3.1:1775764888.354258:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0a00. 00010000:00000010:2.1:1775764888.398554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894c00. 00010000:00000010:2.1:1775764888.415297:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894600. 00010000:00000010:1.1:1775764888.438406:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1600. 00000080:00000010:1.1:1775764888.438441:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8360. 00010000:00000010:1.1:1775764888.459083:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1c00. 00010000:00000010:2.1:1775764888.466276:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966900. 00010000:00000010:2.1:1775764888.466293:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895e00. 00000080:00000010:2.1:1775764888.486953:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2ec0. 00000080:00000010:2.1:1775764888.486987:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275bd280. 00010000:00000010:2.1:1775764888.501222:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899600. 00000080:00000010:1.1:1775764888.547916:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a820. 00010000:00000010:1.1:1775764888.547933:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1000. 00010000:00000010:0.1:1775764888.554029:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0000. 00010000:00000010:0.1:1775764888.554036:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894600. 00000080:00000010:2.1:1775764888.578495:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2ec0. 00010000:00000010:2.1:1775764888.578509:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895e00. 00000080:00000010:2.1:1775764888.609207:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a5c0. 00010000:00000010:3.1:1775764888.630724:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0200. 00000080:00000010:2.1:1775764888.648225:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b400. 00000080:00000010:1.1:1775764888.648322:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6aa40. 00010000:00000010:1.1:1775764888.648335:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894400. 00010000:00000010:0.1:1775764888.686603:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895e00. 00010000:00000010:3.1:1775764888.717171:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0a00. 00000080:00000010:2.1:1775764888.728176:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2560. 00010000:00000010:2.1:1775764888.728203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894400. 00010000:00000010:3.1:1775764888.728580:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894600. 00000080:00000010:2.1:1775764888.767879:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db23c0. 00000080:00000010:3.1:1775764888.773927:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2ec0. 00010000:00000010:0.1:1775764888.797012:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898400. 00000080:00000010:2.1:1775764888.800298:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bf00. 00000080:00000010:3.1:1775764888.828362:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b360. 00000080:00000010:3.1:1775764888.841436:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b7a0. 00010000:00000010:3.1:1775764888.841452:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899000. 00010000:00000010:3.1:1775764888.841460:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d0b500. 00010000:00000010:3.1:1775764888.841466:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903c00. 00010000:00000010:2.1:1775764888.856602:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894400. 00000080:00000010:2.1:1775764888.865293:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b260. 00000080:00000010:2.1:1775764888.865326:0:14454:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275ba100. 00010000:00000010:3.1:1775764888.876223:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894600. 00010000:00000010:3.1:1775764888.888203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895e00. 00000080:00000010:0.1:1775764888.903228:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db23c0. 00010000:00000010:3.1:1775764888.929201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894400. 00010000:00000010:3.1:1775764888.929208:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1c00. 00000080:00000010:0.1:1775764888.949334:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8d00. 00000080:00000010:3.1:1775764888.960857:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2220. 00000080:00000010:2.1:1775764888.965202:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b4e0. 00010000:00000010:2.1:1775764888.965225:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0e00. 00010000:00000010:2.1:1775764888.965230:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d0b300. 00010000:00000010:2.1:1775764888.965233:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994203fb1600. 00010000:00000010:0.1:1775764888.969259:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899000. 00010000:00000010:1.1:1775764888.973569:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895e00. 00010000:00000010:1.1:1775764888.973898:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942079b7000. 00010000:00000010:1.1:1775764888.973904:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898c00. 00000080:00000010:3.1:1775764889.006165:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b7a0. 00000080:00000010:2.1:1775764889.010708:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8640. 00000080:00000010:2.1:1775764889.010742:0:14454:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275d9080. 00000080:00000010:3.1:1775764889.061435:0:32:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275dd280. 00010000:00000010:1.1:1775764889.078134:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894600. 00010000:00000010:3.1:1775764889.096947:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898800. 00010000:00000010:1.1:1775764889.115832:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895e00. 00000080:00000010:1.1:1775764889.126769:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2700. 00010000:00000010:2.1:1775764889.137192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7200. 00010000:00000010:1.1:1775764889.138444:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0a00. 00010000:00000010:2.1:1775764889.140237:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899400. 00010000:00000010:2.1:1775764889.140247:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942069a5f00. 00010000:00000010:2.1:1775764889.140251:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6400. 00010000:00000010:0.1:1775764889.146239:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894400. 00000080:00000010:2.1:1775764889.202200:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a760. 00000080:00000010:2.1:1775764889.211540:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275ddac0. 00010000:00000010:2.1:1775764889.221203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0000. 00010000:00000010:2.1:1775764889.246363:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902800. 00010000:00000010:0.1:1775764889.260243:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894400. 00010000:00000010:3.1:1775764889.294277:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903200. 00010000:00000010:2.1:1775764889.310890:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1400. 00010000:00000010:2.1:1775764889.340860:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0e00. 00010000:00000010:3.1:1775764889.345528:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894600. 00010000:00000010:2.1:1775764889.367186:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1200. 00010000:00000010:0.1:1775764889.375342:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898800. 00010000:00000010:2.1:1775764889.386049:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6400. 00010000:00000010:3.1:1775764889.401768:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895e00. 00010000:00000010:3.1:1775764889.401776:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898600. 00010000:00000010:1.1:1775764889.420278:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899400. 00010000:00000010:3.1:1775764889.439197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899e00. 00010000:00000010:3.1:1775764889.460197:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894400. 00010000:00000010:2.1:1775764889.471221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895e00. 00000080:00000010:2.1:1775764889.471236:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2300. 00010000:00000010:2.1:1775764889.486219:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc800. 00010000:00000010:2.1:1775764889.486239:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894400. 00000080:00000010:2.1:1775764889.486245:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2040. 00010000:00000010:2.1:1775764889.521271:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899e00. 00010000:00000010:2.1:1775764889.557203:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894400. 00010000:00000010:3.1:1775764889.564212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fdc00. 00000080:00000010:2.1:1775764889.570229:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db27e0. 00010000:00000010:3.1:1775764889.582363:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc800. 00010000:00000010:2.1:1775764889.589014:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898600. 00010000:00000010:3.1:1775764889.593517:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd200. 00010000:00000010:2.1:1775764889.596997:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0a00. 00010000:00000010:0.1:1775764889.612196:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1000. 00010000:00000010:1.1:1775764889.634195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899000. 00000080:00000010:0.1:1775764889.634217:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2ce0. 00010000:00000010:2.1:1775764889.641202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1000. 00010000:00000010:3.1:1775764889.644617:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fce00. 00010000:00000010:3.1:1775764889.655744:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fda00. 00010000:00000010:2.1:1775764889.687546:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fde00. 00010000:00000010:3.1:1775764889.713321:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1c00. 00010000:00000010:0.1:1775764889.724632:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fcc00. 00010000:00000010:3.1:1775764889.734804:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2600. 00010000:00000010:3.1:1775764889.734812:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fde00. 00010000:00000010:2.1:1775764889.752142:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb3a00. 00010000:00000010:2.1:1775764889.764247:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898400. 00010000:00000010:2.1:1775764889.775335:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd600. 00010000:00000010:0.1:1775764889.782336:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1c00. 00010000:00000010:2.1:1775764889.805479:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0200. 00010000:00000010:3.1:1775764889.817320:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994203fb0000. 00010000:00000010:2.1:1775764889.832087:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0800. 00010000:00000010:2.1:1775764889.836410:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd600. 00010000:00000010:2.1:1775764889.846362:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1000. 00010000:00000010:2.1:1775764889.871719:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd600. 00010000:00000010:3.1:1775764889.903279:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1e00. 00010000:00000010:2.1:1775764889.909230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fde00. 00010000:00000010:3.1:1775764889.913310:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd600. 00010000:00000010:2.1:1775764889.916512:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994203fb0600. 00010000:00000010:3.1:1775764889.920569:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899600. 00000080:00000010:2.1:1775764889.922059:0:26:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2cc0. 00000080:00000010:2.1:1775764889.922068:0:26:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2e60. 00000080:00000010:2.1:1775764889.922072:0:26:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2460. 00000080:00000010:2.1:1775764889.922078:0:26:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2040. 00010000:00000010:2.1:1775764889.922083:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894600. 00010000:00000010:3.1:1775764889.931271:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1000. 00010000:00000010:2.1:1775764889.956662:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fce00. 00010000:00000010:3.1:1775764889.979286:0:32:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942069a5f00. 00010000:00000010:3.1:1775764889.979297:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fde00. 00010000:00000010:2.1:1775764889.979352:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd200. 00010000:00000010:1.1:1775764889.983267:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fce00. 00010000:00000010:3.1:1775764889.996882:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1c00. 00010000:00000010:2.1:1775764890.002187:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1e00. 00000080:00000010:2.1:1775764890.002246:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2cc0. 00000080:00000010:2.1:1775764890.002253:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2e60. 00000080:00000010:2.1:1775764890.002263:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2460. 00000080:00000010:2.1:1775764890.002268:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8320. 00010000:00000010:2.1:1775764890.002274:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895e00. 00010000:00000010:2.1:1775764890.049193:0:6169:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fda00. 00010000:00000010:3.1:1775764890.050060:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207926200. 00010000:00000010:3.1:1775764890.050068:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd200. 00010000:00000010:2.1:1775764890.058766:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fde00. 00000080:00000010:2.1:1775764890.058782:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2e60. 00010000:00000010:2.1:1775764890.066047:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fce00. 00010000:00000010:3.1:1775764890.085717:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc800. 00010000:00000010:2.1:1775764890.092178:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd600. 00010000:00000010:2.1:1775764890.103228:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fde00. 00000080:00000010:0.1:1775764890.103642:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2cc0. 00010000:00000010:3.1:1775764890.114294:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207de4600. 00010000:00000010:3.1:1775764890.114316:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fce00. 00010000:00000010:3.1:1775764890.125902:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789ca00. 00010000:00000010:3.1:1775764890.141673:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899000. 00010000:00000010:2.1:1775764890.176239:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1c00. 00010000:00000010:3.1:1775764890.192565:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994218fe6200. 00010000:00000010:3.1:1775764890.192576:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898600. 00010000:00000010:3.1:1775764890.213641:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fda00. 00000080:00000010:3.1:1775764890.231471:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2e60. 00010000:00000010:0.1:1775764890.242559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898600. 00010000:00000010:2.1:1775764890.277896:0:14473:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd600. 00010000:00000010:2.1:1775764890.284232:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fde00. 00000080:00000010:2.1:1775764890.311225:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bda0. 00010000:00000010:2.1:1775764890.323704:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fde00. 00010000:00000010:3.1:1775764890.347413:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fce00. 00010000:00000010:3.1:1775764890.365617:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1e00. 00010000:00000010:3.1:1775764890.418917:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899400. 00010000:00000010:3.1:1775764890.418937:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fde00. 00000080:00000010:2.1:1775764890.441937:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2680. 00010000:00000010:3.1:1775764890.470597:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789ca00. 00000080:00000010:0.1:1775764890.474778:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c83c0. 00010000:00000010:2.1:1775764890.474871:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd600. 00000080:00000010:2.1:1775764890.474880:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2180. 00010000:00000010:2.1:1775764890.479182:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fce00. 00010000:00000010:2.1:1775764890.491230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fda00. 00010000:00000010:1.1:1775764890.491263:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1200. 00000080:00000010:1.1:1775764890.491272:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6aac0. 00000080:00000010:2.1:1775764890.491285:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2680. 00010000:00000010:2.1:1775764890.491291:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fde00. 00010000:00000010:2.1:1775764890.516402:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078adc00. 00010000:00000010:2.1:1775764890.516409:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895e00. 00010000:00000010:2.1:1775764890.523226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899400. 00010000:00000010:2.1:1775764890.538275:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895e00. 00000080:00000010:2.1:1775764890.538322:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b3a0. 00010000:00000010:2.1:1775764890.538329:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0e00. 00010000:00000010:3.1:1775764890.544412:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789c000. 00010000:00000010:3.1:1775764890.567747:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894600. 00010000:00000010:3.1:1775764890.572525:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078adc00. 00010000:00000010:3.1:1775764890.572540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894400. 00000080:00000010:1.1:1775764890.576280:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2320. 00010000:00000010:2.1:1775764890.594575:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fce00. 00010000:00000010:2.1:1775764890.601124:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0800. 00010000:00000010:3.1:1775764890.616295:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1e00. 00010000:00000010:2.1:1775764890.631819:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fde00. 00010000:00000010:3.1:1775764890.636241:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0e00. 00000080:00000010:2.1:1775764890.645656:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2680. 00010000:00000010:2.1:1775764890.645674:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789da00. 00000080:00000010:2.1:1775764890.645678:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8980. 00010000:00000010:3.1:1775764890.655296:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894600. 00010000:00000010:3.1:1775764890.673255:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd600. 00010000:00000010:2.1:1775764890.690181:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fce00. 00000080:00000010:2.1:1775764890.701186:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b1c0. 00010000:00000010:3.1:1775764890.702204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fde00. 00010000:00000010:3.1:1775764890.716326:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917200. 00010000:00000010:0.1:1775764890.718455:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789c800. 00000080:00000010:2.1:1775764890.726822:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2aa0. 00000080:00000010:1.1:1775764890.727332:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8300. 00000080:00000010:2.1:1775764890.733542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a260. 00000080:00000010:2.1:1775764890.745213:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2440. 00000080:00000010:2.1:1775764890.749608:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a780. 00000080:00000010:2.1:1775764890.756215:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2300. 00010000:00000010:3.1:1775764890.757287:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898600. 00000080:00000010:2.1:1775764890.777223:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b500. 00000080:00000010:3.1:1775764890.777327:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2b20. 00010000:00000010:0.1:1775764890.780326:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916800. 00000080:00000010:2.1:1775764890.781190:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db21a0. 00000080:00000010:3.1:1775764890.790343:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8620. 00000080:00000010:3.1:1775764890.790641:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2680. 00010000:00000010:3.1:1775764890.790651:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1000. 00010000:00000010:3.1:1775764890.790657:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942079a4100. 00010000:00000010:3.1:1775764890.790663:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899000. 00010000:00000010:3.1:1775764890.790667:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895e00. 00000080:00000010:3.1:1775764890.790671:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db28e0. 00000080:00000010:3.1:1775764890.790684:0:14454:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff99422762a100. 00000080:00000010:2.1:1775764890.793530:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2b00. 00000080:00000010:2.1:1775764890.798858:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b360. 00000080:00000010:3.1:1775764890.806122:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b5e0. 00010000:00000010:2.1:1775764890.814299:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fdc00. 00010000:00000010:2.1:1775764890.814307:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207de4c00. 00010000:00000010:2.1:1775764890.814312:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0a00. 00000080:00000010:2.1:1775764890.819663:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bb60. 00000080:00000010:2.1:1775764890.819692:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275ba100. 00000080:00000010:3.1:1775764890.820325:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2bc0. 00000080:00000010:3.1:1775764890.828304:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c81e0. 00010000:00000010:2.1:1775764890.837620:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917600. 00000080:00000010:2.1:1775764890.837627:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2b00. 00000080:00000010:2.1:1775764890.837637:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a420. 00000080:00000010:0.1:1775764890.837705:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2e40. 00000080:00000010:0.1:1775764890.846081:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a440. 00000080:00000010:0.1:1775764890.846134:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b380. 00010000:00000010:0.1:1775764890.846142:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1c00. 00010000:00000010:0.1:1775764890.846147:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ada00. 00010000:00000010:0.1:1775764890.846151:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc800. 00010000:00000010:0.1:1775764890.846178:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0200. 00000080:00000010:0.1:1775764890.864282:0:2400:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db27e0. 00000080:00000010:0.1:1775764890.864300:0:2400:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275ba940. 00010000:00000010:2.1:1775764890.885558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898400. 00010000:00000010:2.1:1775764890.885568:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942079b7100. 00010000:00000010:2.1:1775764890.885571:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00010000:00000010:0.1:1775764890.885618:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789da00. 00000080:00000010:3.1:1775764890.888418:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275bd280. 00000080:00000010:2.1:1775764890.894225:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6aa00. 00010000:00000010:3.1:1775764890.906707:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898c00. 00010000:00000010:0.1:1775764890.943362:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789c800. 00000080:00000010:0.1:1775764890.943371:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b3e0. 00010000:00000010:3.1:1775764890.947438:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899000. 00010000:00000010:2.1:1775764890.989206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917600. 00010000:00000010:1.1:1775764891.015473:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917000. 00000080:00000010:2.1:1775764891.015834:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c82e0. 00010000:00000010:2.1:1775764891.015847:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789d400. 00010000:00000010:3.1:1775764891.016790:0:14473:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1400. 00010000:00000010:3.1:1775764891.027588:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1200. 00010000:00000010:3.1:1775764891.053189:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916600. 00010000:00000010:3.1:1775764891.065917:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917600. 00010000:00000010:0.1:1775764891.094434:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899000. 00010000:00000010:3.1:1775764891.099960:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc400. 00010000:00000010:2.1:1775764891.101545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789c400. 00000080:00000010:3.1:1775764891.114151:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8620. 00010000:00000010:1.1:1775764891.120720:0:6158:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898600. 00010000:00000010:2.1:1775764891.138520:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898c00. 00010000:00000010:3.1:1775764891.143254:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917e00. 00010000:00000010:2.1:1775764891.166066:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789dc00. 00010000:00000010:3.1:1775764891.178708:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc000. 00010000:00000010:3.1:1775764891.199984:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789d000. 00010000:00000010:2.1:1775764891.203236:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00010000:00000010:3.1:1775764891.223547:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917a00. 00010000:00000010:3.1:1775764891.223553:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd000. 00010000:00000010:2.1:1775764891.230712:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916c00. 00010000:00000010:3.1:1775764891.241317:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789ce00. 00010000:00000010:3.1:1775764891.245199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc600. 00010000:00000010:3.1:1775764891.258561:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916c00. 00010000:00000010:2.1:1775764891.270289:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916000. 00010000:00000010:3.1:1775764891.275869:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1000. 00010000:00000010:3.1:1775764891.287231:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916a00. 00010000:00000010:0.1:1775764891.292228:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789ca00. 00010000:00000010:0.1:1775764891.307208:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916000. 00010000:00000010:3.1:1775764891.326332:0:31:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898400. 00010000:00000010:1.1:1775764891.356544:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917800. 00010000:00000010:3.1:1775764891.358533:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00010000:00000010:3.1:1775764891.358539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fcc00. 00010000:00000010:2.1:1775764891.370818:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916000. 00000080:00000010:0.1:1775764891.372500:0:12:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b3c0. 00000080:00000010:0.1:1775764891.372524:0:12:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2b00. 00010000:00000010:0.1:1775764891.372530:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916400. 00010000:00000010:1.1:1775764891.414251:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00010000:00000010:0.1:1775764891.438209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916000. 00010000:00000010:1.1:1775764891.448224:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898400. 00010000:00000010:2.1:1775764891.455571:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ad100. 00010000:00000010:2.1:1775764891.455577:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789ca00. 00010000:00000010:1.1:1775764891.456525:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1000. 00000080:00000010:1.1:1775764891.456567:0:2392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a080. 00000080:00000010:1.1:1775764891.456575:0:2392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8320. 00010000:00000010:1.1:1775764891.456580:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789da00. 00010000:00000010:0.1:1775764891.480794:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917a00. 00010000:00000010:3.1:1775764891.499834:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916c00. 00010000:00000010:3.1:1775764891.507482:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942079b7100. 00010000:00000010:3.1:1775764891.507498:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1e00. 00000080:00000010:2.1:1775764891.509538:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6ba60. 00010000:00000010:0.1:1775764891.523275:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916800. 00010000:00000010:2.1:1775764891.553221:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917e00. 00010000:00000010:2.1:1775764891.563432:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917000. 00010000:00000010:2.1:1775764891.569898:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1400. 00010000:00000010:3.1:1775764891.585359:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898c00. 00000080:00000010:2.1:1775764891.605247:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8980. 00010000:00000010:3.1:1775764891.615478:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789dc00. 00000080:00000010:2.1:1775764891.633970:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6ba40. 00010000:00000010:1.1:1775764891.637429:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916c00. 00010000:00000010:2.1:1775764891.661261:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917200. 00010000:00000010:3.1:1775764891.680142:0:6169:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207de4200. 00010000:00000010:3.1:1775764891.680149:0:6169:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899e00. 00010000:00000010:3.1:1775764891.680168:0:6169:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d0b100. 00010000:00000010:3.1:1775764891.680173:0:6169:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6600. 00010000:00000010:3.1:1775764891.680176:0:6169:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942069a5400. 00010000:00000010:3.1:1775764891.680180:0:6169:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894c00. 00010000:00000010:2.1:1775764891.682208:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966e00. 00010000:00000010:2.1:1775764891.682216:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7800. 00010000:00000010:2.1:1775764891.682220:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994211cc1000. 00010000:00000010:2.1:1775764891.682222:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6c00. 00010000:00000010:3.1:1775764891.693175:0:32:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942079b7400. 00010000:00000010:3.1:1775764891.693202:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7e00. 00010000:00000010:3.1:1775764891.693206:0:32:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966b00. 00010000:00000010:3.1:1775764891.693210:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894200. 00010000:00000010:3.1:1775764891.693213:0:32:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d0bc00. 00010000:00000010:3.1:1775764891.693216:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903a00. 00010000:00000010:3.1:1775764891.693221:0:32:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207de4b00. 00010000:00000010:3.1:1775764891.693225:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902000. 00010000:00000010:2.1:1775764891.707702:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917a00. 00010000:00000010:3.1:1775764891.749730:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789ce00. 00000080:00000010:3.1:1775764891.749750:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8020. 00000080:00000010:2.1:1775764891.759643:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db29a0. 00010000:00000010:3.1:1775764891.771211:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917a00. 00010000:00000010:3.1:1775764891.784332:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917200. 00010000:00000010:0.1:1775764891.809484:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789dc00. 00000080:00000010:1.1:1775764891.825267:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db29a0. 00010000:00000010:1.1:1775764891.825288:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789da00. 00010000:00000010:2.1:1775764891.841297:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916000. 00000080:00000010:3.1:1775764891.841732:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db20a0. 00010000:00000010:3.1:1775764891.888296:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966b00. 00010000:00000010:3.1:1775764891.888310:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1200. 00010000:00000010:2.1:1775764891.910226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898400. 00010000:00000010:3.1:1775764891.917623:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916000. 00000080:00000010:3.1:1775764891.917714:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b300. 00010000:00000010:3.1:1775764891.917723:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898c00. 00010000:00000010:2.1:1775764891.921368:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898600. 00010000:00000010:2.1:1775764891.984449:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899e00. 00010000:00000010:3.1:1775764892.017263:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966b00. 00010000:00000010:3.1:1775764892.017270:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00000080:00000010:3.1:1775764892.029379:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c81e0. 00010000:00000010:3.1:1775764892.049349:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1e00. 00010000:00000010:3.1:1775764892.049360:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917200. 00010000:00000010:2.1:1775764892.063234:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916400. 00010000:00000010:3.1:1775764892.074447:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917800. 00010000:00000010:2.1:1775764892.087225:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0200. 00000080:00000010:1.1:1775764892.107850:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2180. 00010000:00000010:1.1:1775764892.107871:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0e00. 00010000:00000010:0.1:1775764892.112422:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916400. 00000080:00000010:0.1:1775764892.112443:0:2392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2280. 00010000:00000010:2.1:1775764892.120034:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917a00. 00010000:00000010:3.1:1775764892.135276:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916c00. 00010000:00000010:0.1:1775764892.147545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916400. 00010000:00000010:2.1:1775764892.159240:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789dc00. 00000080:00000010:0.1:1775764892.163881:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db23e0. 00010000:00000010:0.1:1775764892.163901:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789ca00. 00010000:00000010:3.1:1775764892.173853:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0e00. 00000080:00000010:3.1:1775764892.173876:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a900. 00010000:00000010:3.1:1775764892.222705:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898c00. 00010000:00000010:3.1:1775764892.222727:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207de4900. 00010000:00000010:3.1:1775764892.222733:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789ce00. 00010000:00000010:1.1:1775764892.253528:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917200. 00000080:00000010:1.1:1775764892.257657:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bb40. 00010000:00000010:1.1:1775764892.260242:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00010000:00000010:1.1:1775764892.260261:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917800. 00010000:00000010:2.1:1775764892.321865:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0200. 00010000:00000010:3.1:1775764892.335211:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943000. 00010000:00000010:3.1:1775764892.335221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0e00. 00000080:00000010:2.1:1775764892.349212:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c82e0. 00010000:00000010:2.1:1775764892.353312:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917200. 00010000:00000010:3.1:1775764892.385218:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00010000:00000010:3.1:1775764892.385225:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916400. 00010000:00000010:2.1:1775764892.420756:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898c00. 00010000:00000010:2.1:1775764892.446252:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899e00. 00010000:00000010:1.1:1775764892.450366:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fcc00. 00000080:00000010:2.1:1775764892.450531:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2480. 00010000:00000010:2.1:1775764892.450549:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6e00. 00000080:00000010:2.1:1775764892.450552:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a780. 00010000:00000010:3.1:1775764892.467647:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7c00. 00010000:00000010:2.1:1775764892.516731:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00010000:00000010:2.1:1775764892.516745:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd000. 00000080:00000010:0.1:1775764892.528711:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2ea0. 00010000:00000010:0.1:1775764892.542211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789da00. 00000080:00000010:3.1:1775764892.552132:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2480. 00010000:00000010:3.1:1775764892.552143:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917200. 00010000:00000010:3.1:1775764892.552150:0:32:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078efb00. 00010000:00000010:3.1:1775764892.552197:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916c00. 00010000:00000010:3.1:1775764892.552201:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc600. 00010000:00000010:2.1:1775764892.552441:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc000. 00000080:00000010:2.1:1775764892.552447:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2fa0. 00000080:00000010:3.1:1775764892.584414:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8920. 00000080:00000010:3.1:1775764892.584464:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227629080. 00010000:00000010:1.1:1775764892.595320:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc000. 00000080:00000010:2.1:1775764892.596200:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2fa0. 00010000:00000010:3.1:1775764892.596589:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898400. 00000080:00000010:1.1:1775764892.605884:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2500. 00010000:00000010:2.1:1775764892.609694:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc400. 00000080:00000010:2.1:1775764892.609702:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db24e0. 00010000:00000010:3.1:1775764892.614326:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7c00. 00000080:00000010:1.1:1775764892.625245:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b980. 00010000:00000010:1.1:1775764892.625256:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898600. 00010000:00000010:1.1:1775764892.625261:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942079b7300. 00010000:00000010:1.1:1775764892.625265:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7000. 00010000:00000010:1.1:1775764892.625268:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd000. 00000080:00000010:1.1:1775764892.631279:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a5c0. 00000080:00000010:1.1:1775764892.635790:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff994227606b40. 00010000:00000010:2.1:1775764892.646670:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc400. 00010000:00000010:3.1:1775764892.647851:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7000. 00000080:00000010:1.1:1775764892.651741:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6aa00. 00010000:00000010:1.1:1775764892.651754:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6a00. 00000080:00000010:1.1:1775764892.651757:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a5c0. 00010000:00000010:2.1:1775764892.659600:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fdc00. 00010000:00000010:2.1:1775764892.677238:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789c000. 00010000:00000010:3.1:1775764892.705216:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fdc00. 00010000:00000010:3.1:1775764892.713200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898400. 00010000:00000010:3.1:1775764892.713210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6a00. 00010000:00000010:3.1:1775764892.713212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc400. 00000080:00000010:2.1:1775764892.717273:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2880. 00010000:00000010:3.1:1775764892.718247:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd600. 00010000:00000010:2.1:1775764892.729647:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789c000. 00010000:00000010:3.1:1775764892.730215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916800. 00000080:00000010:0.1:1775764892.741536:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8520. 00010000:00000010:3.1:1775764892.757537:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789ce00. 00010000:00000010:3.1:1775764892.772417:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fde00. 00010000:00000010:2.1:1775764892.779326:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898400. 00010000:00000010:2.1:1775764892.779351:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789d000. 00000080:00000010:3.1:1775764892.779452:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b7c0. 00010000:00000010:3.1:1775764892.801625:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fce00. 00000080:00000010:2.1:1775764892.806576:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c84a0. 00000080:00000010:0.1:1775764892.827227:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2360. 00000080:00000010:2.1:1775764892.827368:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b5e0. 00010000:00000010:3.1:1775764892.841581:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd600. 00000080:00000010:2.1:1775764892.849200:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8300. 00000080:00000010:2.1:1775764892.858054:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8640. 00000080:00000010:2.1:1775764892.866909:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b020. 00000080:00000010:2.1:1775764892.880296:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2aa0. 00000080:00000010:0.1:1775764892.883233:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2d80. 00000080:00000010:2.1:1775764892.884259:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2fa0. 00000080:00000010:2.1:1775764892.884270:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db24e0. 00010000:00000010:2.1:1775764892.884277:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7c00. 00010000:00000010:2.1:1775764892.884284:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078adc00. 00010000:00000010:2.1:1775764892.884291:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917600. 00010000:00000010:2.1:1775764892.884296:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc800. 00000080:00000010:2.1:1775764892.888219:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2d60. 00000080:00000010:2.1:1775764892.888245:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275ba100. 00000080:00000010:1.1:1775764892.888406:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2b80. 00010000:00000010:2.1:1775764892.892213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc400. 00000080:00000010:2.1:1775764892.896214:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2240. 00000080:00000010:1.1:1775764892.901975:0:20:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bb20. 00000080:00000010:1.1:1775764892.901984:0:20:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2f60. 00010000:00000010:1.1:1775764892.901990:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fde00. 00010000:00000010:1.1:1775764892.902010:0:20:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ad100. 00010000:00000010:1.1:1775764892.902015:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917e00. 00010000:00000010:1.1:1775764892.902019:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7a00. 00000080:00000010:2.1:1775764892.905256:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2900. 00000080:00000010:2.1:1775764892.910339:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9942275dd280. 00010000:00000010:2.1:1775764892.919760:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc400. 00000080:00000010:2.1:1775764892.919769:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2240. 00010000:00000010:3.1:1775764892.923226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd000. 00010000:00000010:2.1:1775764892.930721:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798ce00. 00010000:00000010:3.1:1775764892.950219:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789d000. 00010000:00000010:3.1:1775764892.966520:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc400. 00010000:00000010:3.1:1775764892.970386:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420694dc00. 00010000:00000010:1.1:1775764892.974884:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6e00. 00000080:00000010:1.1:1775764892.984270:0:2392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2900. 00010000:00000010:3.1:1775764892.990230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6600. 00010000:00000010:3.1:1775764892.999440:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc000. 00010000:00000010:3.1:1775764893.007203:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789da00. 00010000:00000010:3.1:1775764893.029700:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fcc00. 00010000:00000010:2.1:1775764893.038317:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc000. 00000080:00000010:3.1:1775764893.055783:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2240. 00010000:00000010:3.1:1775764893.062702:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898600. 00010000:00000010:3.1:1775764893.062726:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7a00. 00010000:00000010:2.1:1775764893.065196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc600. 00010000:00000010:2.1:1775764893.065203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079cbe00. 00010000:00000010:3.1:1775764893.083247:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc000. 00010000:00000010:2.1:1775764893.085184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7c00. 00010000:00000010:2.1:1775764893.104211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789da00. 00010000:00000010:0.1:1775764893.125003:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc400. 00010000:00000010:2.1:1775764893.130699:0:6169:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420794c900. 00010000:00000010:2.1:1775764893.130708:0:6169:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079cbc00. 00010000:00000010:2.1:1775764893.138353:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420694c400. 00010000:00000010:3.1:1775764893.149518:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd000. 00010000:00000010:2.1:1775764893.165375:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fde00. 00010000:00000010:2.1:1775764893.165395:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fce00. 00010000:00000010:2.1:1775764893.190207:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207976000. 00010000:00000010:2.1:1775764893.190224:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798d800. 00010000:00000010:2.1:1775764893.198278:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898600. 00010000:00000010:2.1:1775764893.202359:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207976500. 00010000:00000010:2.1:1775764893.202366:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079cae00. 00010000:00000010:2.1:1775764893.214124:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fce00. 00010000:00000010:3.1:1775764893.226205:0:6168:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00010000:00000010:2.1:1775764893.236213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd000. 00010000:00000010:3.1:1775764893.251305:0:2402:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078adf00. 00010000:00000010:3.1:1775764893.251318:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207535e00. 00010000:00000010:0.1:1775764893.265970:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7c00. 00010000:00000010:2.1:1775764893.270240:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fce00. 00010000:00000010:0.1:1775764893.288227:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd000. 00010000:00000010:3.1:1775764893.288277:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ade00. 00010000:00000010:3.1:1775764893.288283:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb3200. 00010000:00000010:3.1:1775764893.296318:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420550b300. 00010000:00000010:3.1:1775764893.296343:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798cc00. 00010000:00000010:2.1:1775764893.314398:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6400. 00010000:00000010:3.1:1775764893.322831:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fce00. 00010000:00000010:1.1:1775764893.347597:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc000. 00010000:00000010:0.1:1775764893.357207:0:2401:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7a00. 00010000:00000010:2.1:1775764893.368287:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207976100. 00010000:00000010:2.1:1775764893.368304:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079cb400. 00010000:00000010:1.1:1775764893.376641:0:2392:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ef300. 00010000:00000010:1.1:1775764893.376654:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cbce00. 00010000:00000010:3.1:1775764893.384206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fcc00. 00010000:00000010:2.1:1775764893.399445:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898c00. 00010000:00000010:3.1:1775764893.411465:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7e00. 00010000:00000010:0.1:1775764893.419193:0:6158:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420550b800. 00010000:00000010:3.1:1775764893.419198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917e00. 00010000:00000010:0.1:1775764893.419200:0:6158:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb9e00. 00010000:00000010:2.1:1775764893.426462:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207976600. 00010000:00000010:2.1:1775764893.426476:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0600. 00010000:00000010:3.1:1775764893.440082:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fcc00. 00010000:00000010:3.1:1775764893.452394:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc000. 00010000:00000010:1.1:1775764893.459201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898c00. 00010000:00000010:3.1:1775764893.459494:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789da00. 00010000:00000010:3.1:1775764893.459502:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420550bd00. 00010000:00000010:3.1:1775764893.459506:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0c00. 00010000:00000010:2.1:1775764893.478350:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ad700. 00010000:00000010:2.1:1775764893.478365:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cbcc00. 00010000:00000010:0.1:1775764893.491554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917e00. 00010000:00000010:3.1:1775764893.503273:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789c000. 00010000:00000010:1.1:1775764893.513711:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994209db3000. 00010000:00000010:2.1:1775764893.519198:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ef200. 00010000:00000010:2.1:1775764893.519205:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970c00. 00010000:00000010:3.1:1775764893.533527:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898c00. 00010000:00000010:1.1:1775764893.552218:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917600. 00010000:00000010:2.1:1775764893.552218:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ef400. 00010000:00000010:2.1:1775764893.552225:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cbca00. 00010000:00000010:0.1:1775764893.569491:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916800. 00010000:00000010:0.1:1775764893.569513:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994209db2a00. 00010000:00000010:0.1:1775764893.599257:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420550ba00. 00010000:00000010:0.1:1775764893.599266:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1c00. 00010000:00000010:1.1:1775764893.619296:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970a00. 00010000:00000010:3.1:1775764893.626413:0:2403:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ef900. 00010000:00000010:3.1:1775764893.626423:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ce1200. 00010000:00000010:2.1:1775764893.634206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898c00. 00010000:00000010:2.1:1775764893.638208:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80800. 00010000:00000010:1.1:1775764893.652228:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971800. 00010000:00000010:3.1:1775764893.656280:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ef600. 00010000:00000010:3.1:1775764893.656299:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ce1600. 00010000:00000010:3.1:1775764893.668514:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80800. 00010000:00000010:2.1:1775764893.677302:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ad300. 00010000:00000010:2.1:1775764893.677316:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0c00. 00010000:00000010:2.1:1775764893.689454:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970600. 00010000:00000010:3.1:1775764893.707127:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81800. 00010000:00000010:3.1:1775764893.716428:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7e00. 00010000:00000010:2.1:1775764893.727758:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078efe00. 00010000:00000010:2.1:1775764893.727766:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075dae00. 00010000:00000010:1.1:1775764893.748241:0:1358:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80000. 00010000:00000010:2.1:1775764893.755366:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971400. 00010000:00000010:3.1:1775764893.768954:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078eff00. 00010000:00000010:3.1:1775764893.768963:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075da200. 00010000:00000010:3.1:1775764893.773271:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80400. 00010000:00000010:2.1:1775764893.788052:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916c00. 00010000:00000010:2.1:1775764893.795331:0:14453:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420550b400. 00010000:00000010:2.1:1775764893.795339:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81a00. 00010000:00000010:0.1:1775764893.807211:0:6158:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6400. 00010000:00000010:3.1:1775764893.824542:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81400. 00010000:00000010:0.1:1775764893.829371:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971000. 00000080:00000010:0.1:1775764893.829435:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f71560. 00000080:00000010:0.1:1775764893.829443:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a900. 00010000:00000010:0.1:1775764893.829449:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc800. 00010000:00000010:0.1:1775764893.881201:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81000. 00010000:00000010:2.1:1775764893.889654:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420550b200. 00010000:00000010:2.1:1775764893.889663:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81c00. 00010000:00000010:3.1:1775764893.900353:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81200. 00010000:00000010:2.1:1775764893.903138:0:2403:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207de4800. 00010000:00000010:2.1:1775764893.903149:0:2403:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970600. 00010000:00000010:0.1:1775764893.922489:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970e00. 00010000:00000010:3.1:1775764893.930200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80c00. 00010000:00000010:2.1:1775764893.942599:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971a00. 00000080:00000010:2.1:1775764893.948024:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f71cc0. 00000080:00000010:2.1:1775764893.948038:0:14453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db24e0. 00010000:00000010:2.1:1775764893.948046:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fdc00. 00010000:00000010:3.1:1775764893.960239:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80600. 00000080:00000010:3.1:1775764893.990370:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6aac0. 00010000:00000010:2.1:1775764893.994122:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078efd00. 00010000:00000010:2.1:1775764893.994132:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80c00. 00010000:00000010:3.1:1775764893.994231:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916c00. 00010000:00000010:3.1:1775764894.009248:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917200. 00010000:00000010:3.1:1775764894.017547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81400. 00010000:00000010:3.1:1775764894.039872:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80400. 00010000:00000010:3.1:1775764894.051016:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81000. 00010000:00000010:3.1:1775764894.086447:0:31:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970400. 00000080:00000010:0.1:1775764894.087198:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f71880. 00000080:00000010:3.1:1775764894.103417:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f71dc0. 00010000:00000010:2.1:1775764894.104488:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917200. 00010000:00000010:3.1:1775764894.119937:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970e00. 00000080:00000010:3.1:1775764894.119954:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f71b20. 00010000:00000010:2.1:1775764894.124011:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80000. 00010000:00000010:3.1:1775764894.155877:0:14473:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80800. 00000080:00000010:2.1:1775764894.167384:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c84a0. 00010000:00000010:3.1:1775764894.172211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81000. 00000080:00000010:3.1:1775764894.178196:0:2403:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a900. 00010000:00000010:2.1:1775764894.178460:0:14453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6400. 00010000:00000010:3.1:1775764894.188212:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207de4b00. 00010000:00000010:3.1:1775764894.188221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994209db3400. 00010000:00000010:2.1:1775764894.211698:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81400. 00000080:00000010:2.1:1775764894.211707:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2880. 00010000:00000010:2.1:1775764894.222969:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80400. 00000080:00000010:2.1:1775764894.247645:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8640. 00010000:00000010:1.1:1775764894.255259:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7a00. 00010000:00000010:0.1:1775764894.256825:0:2392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80400. 00000080:00000010:2.1:1775764894.262826:0:14454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db2440. 00010000:00000010:2.1:1775764894.262847:0:14454:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ef400. 00010000:00000010:2.1:1775764894.262854:0:14454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80200. 00010000:00000010:3.1:1775764894.279291:0:2402:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80800. 00010000:00000010:0.1:1775764894.296264:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81400. 00010000:00000010:3.1:1775764894.315265:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078efd00. 00010000:00000010:3.1:1775764894.315282:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80600. 00010000:00000010:2.1:1775764894.323216:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80200. 00000080:00000010:3.1:1775764903.671200:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942074727c0. 00000080:00000010:0.1:1775764903.675183:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942074726c0. 00000080:00000010:0.1:1775764903.675191:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c80e0. 00000080:00000010:3.1:1775764903.675193:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472d40. 00000080:00000010:3.1:1775764903.675204:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c8da0. 00000080:00000010:3.1:1775764903.675226:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942069c80c0. 00010000:00000010:3.1:1775764903.675231:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994220375600. 00010000:00000010:3.1:1775764903.675236:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994201483c00. 00010000:00000010:3.1:1775764903.675240:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99421805e800. 00010000:00000010:3.1:1775764903.675243:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942013d3e00. 00010000:00000010:3.1:1775764903.675246:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994220375a00. 00010000:00000010:3.1:1775764903.675249:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994218f6fc00. 00010000:00000010:3.1:1775764903.675252:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994200030000. 00010000:00000010:3.1:1775764903.675254:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994220375000. 00010000:00000010:3.1:1775764903.675257:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994200031200. 00010000:00000010:3.1:1775764903.675260:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99421805e200. 00010000:00000010:3.1:1775764903.675263:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420b14bc00. 00010000:00000010:3.1:1775764903.675265:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075dac00. 00010000:00000010:3.1:1775764903.675268:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994200031600. 00010000:00000010:3.1:1775764903.675270:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994200030200. 00010000:00000010:3.1:1775764903.675272:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420b14be00. 00010000:00000010:3.1:1775764903.675274:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209c33d00. 00010000:00000010:3.1:1775764903.675277:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994200031800. 00010000:00000010:3.1:1775764903.675279:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942013d2200. 00010000:00000010:3.1:1775764903.675282:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209c33e00. 00010000:00000010:3.1:1775764903.675285:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942013d3000. 00010000:00000010:3.1:1775764903.675287:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420b14b400. 00010000:00000010:3.1:1775764903.675289:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209c33800. 00010000:00000010:3.1:1775764903.675291:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942013d2c00. 00010000:00000010:3.1:1775764903.675293:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209c33300. 00010000:00000010:3.1:1775764903.675296:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942013d3200. 00000800:00000100:1.1:1775764955.569230:0:2391:0:(socklnd_cb.c:2416:ksocknal_find_timed_out_conn()) Timeout sending data to 12345-192.168.202.156@tcp (192.168.202.156:988) the network or that node may be down. 00000800:00000200:1.1:1775764955.569235:0:2391:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 3, ksnr_connected 6 ksnr_max_conns 1 00000800:00000200:1.1:1775764955.569239:0:2391:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 2, ksnr_connected 2 ksnr_max_conns 1 00000800:00000200:1.1:1775764955.569240:0:2391:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 1, ksnr_connected 0 ksnr_max_conns 1 00000800:00000200:2.1:1775764956.851601:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775764956.852048:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775764956.852441:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775764956.852725:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775764956.852876:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775764961.969921:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775764961.970464:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775764961.970794:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775764961.970939:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775764961.971033:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775764967.089959:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775764967.090520:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775764967.090871:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775764967.091081:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775764967.091203:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775764972.209784:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775764972.210138:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775764972.210325:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775764972.210409:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775764972.210523:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775764977.329750:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775764977.329776:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 8, 5 secs later) 00000800:00000200:0.1:1775764977.330592:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775764977.330595:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 8, 5 secs later) 00000800:00000200:0.1:1775764977.330870:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775764977.330872:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 8, 5 secs later) 00000800:00000200:0.1:1775764977.331132:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775764977.331135:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 8, 5 secs later) 00000800:00000200:0.1:1775764977.331588:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775764977.331591:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 8, 5 secs later) 00000800:00000200:0.1:1775764982.449906:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775764982.450445:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775764982.450665:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775764982.450758:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775764982.450840:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775764987.569908:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775764987.569913:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 16, 12 secs later) 00000800:00000200:0.1:1775764987.570697:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775764987.570702:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 16, 12 secs later) 00000800:00000200:0.1:1775764987.571176:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775764987.571179:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 16, 12 secs later) 00000800:00000200:0.1:1775764987.571581:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775764987.571584:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 16, 12 secs later) 00000800:00000200:0.1:1775764987.572062:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775764987.572064:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 16, 12 secs later) 00000800:00000200:0.1:1775764992.689837:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775764992.689855:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 16, 7 secs later) 00000800:00000200:0.1:1775764992.690411:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775764992.690414:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 16, 7 secs later) 00000800:00000200:0.1:1775764992.690648:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775764992.690650:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 16, 7 secs later) 00000800:00000200:0.1:1775764992.690864:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775764992.690866:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 16, 7 secs later) 00000800:00000200:0.1:1775764992.691094:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775764992.691096:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 16, 7 secs later) 00000800:00000200:0.1:1775764997.809847:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775764997.809861:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 16, 2 secs later) 00000800:00000200:0.1:1775764997.810492:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775764997.810494:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 16, 2 secs later) 00000800:00000200:0.1:1775764997.810686:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775764997.810688:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 16, 2 secs later) 00000800:00000200:0.1:1775764997.810843:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775764997.810845:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 16, 2 secs later) 00000800:00000200:0.1:1775764997.811127:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775764997.811130:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 16, 2 secs later) 00000800:00000200:0.1:1775765002.929946:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765002.930442:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765002.930881:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765002.931041:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765002.931134:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765008.049803:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765008.049817:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 32, 29 secs later) 00000800:00000200:0.1:1775765008.050415:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765008.050416:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 32, 29 secs later) 00000800:00000200:0.1:1775765008.050619:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765008.050621:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 32, 29 secs later) 00000800:00000200:0.1:1775765008.050822:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765008.050824:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 32, 29 secs later) 00000800:00000200:0.1:1775765008.051123:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765008.051125:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 32, 29 secs later) 00000800:00000200:0.1:1775765013.169904:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765013.169908:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 32, 24 secs later) 00000800:00000200:0.1:1775765013.170715:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765013.170718:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 32, 24 secs later) 00000800:00000200:0.1:1775765013.170991:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765013.170993:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 32, 24 secs later) 00000800:00000200:0.1:1775765013.171345:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765013.171347:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 32, 24 secs later) 00000800:00000200:0.1:1775765013.171598:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765013.171600:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 32, 24 secs later) 00000800:00000200:1.1:1775765018.290444:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765018.290459:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 32, 19 secs later) 00000800:00000200:1.1:1775765018.291854:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765018.291862:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 32, 19 secs later) 00000800:00000200:1.1:1775765018.292808:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765018.292814:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 32, 19 secs later) 00000800:00000200:1.1:1775765018.293693:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765018.293704:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 32, 19 secs later) 00000800:00000200:1.1:1775765018.294438:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765018.294439:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 32, 19 secs later) 00000800:00000200:1.1:1775765023.409785:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765023.409791:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 32, 14 secs later) 00000800:00000200:1.1:1775765023.410448:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765023.410450:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 32, 14 secs later) 00000800:00000200:1.1:1775765023.410639:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765023.410640:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 32, 14 secs later) 00000800:00000200:1.1:1775765023.410896:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765023.410898:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 32, 14 secs later) 00000800:00000200:1.1:1775765023.411195:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765023.411197:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 32, 14 secs later) 00000800:00000200:1.1:1775765028.529860:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765028.529866:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 32, 8 secs later) 00000800:00000200:1.1:1775765028.530475:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765028.530478:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 32, 8 secs later) 00000800:00000200:1.1:1775765028.530784:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765028.530786:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 32, 8 secs later) 00000800:00000200:1.1:1775765028.531072:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765028.531073:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 32, 8 secs later) 00000800:00000200:1.1:1775765028.531292:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765028.531295:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 32, 8 secs later) 00000800:00000200:0.1:1775765033.649852:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765033.649859:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 32, 3 secs later) 00000800:00000200:0.1:1775765033.650629:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765033.650633:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 32, 3 secs later) 00000800:00000200:0.1:1775765033.651031:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765033.651046:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 32, 3 secs later) 00000800:00000200:0.1:1775765033.651339:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765033.651341:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 32, 3 secs later) 00000800:00000200:0.1:1775765033.651557:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765033.651559:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 32, 3 secs later) 00000800:00000200:0.1:1775765038.769903:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765038.770212:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765038.770450:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765038.770622:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765038.770776:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765043.890287:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765043.890293:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775765043.891070:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765043.891074:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775765043.891338:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765043.891340:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775765043.891586:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765043.891588:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775765043.891998:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765043.892000:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775765049.009875:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765049.009882:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775765049.010723:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765049.010746:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775765049.011332:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765049.011337:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775765049.011854:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765049.011859:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775765049.012300:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765049.012301:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775765054.130211:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765054.130223:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775765054.131354:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765054.131356:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775765054.131573:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765054.131587:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775765054.131810:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765054.131812:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775765054.132087:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765054.132089:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775765059.250037:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765059.250046:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775765059.250986:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765059.250988:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775765059.251210:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765059.251211:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775765059.251399:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765059.251401:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775765059.251714:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765059.251716:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775765064.370211:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765064.370219:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775765064.371071:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765064.371072:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775765064.371372:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765064.371373:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775765064.371566:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765064.371568:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775765064.371811:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765064.371829:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775765069.489911:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765069.489917:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775765069.490550:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765069.490552:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775765069.490834:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765069.490836:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775765069.491052:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765069.491054:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775765069.491321:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765069.491323:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775765074.609904:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765074.609910:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775765074.610662:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765074.610663:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775765074.610870:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765074.610872:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775765074.611092:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765074.611094:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775765074.611442:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765074.611444:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775765079.730136:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765079.730143:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775765079.731292:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765079.731314:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775765079.731922:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765079.731925:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775765079.732362:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765079.732364:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775765079.732569:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765079.732571:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775765084.850525:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765084.850537:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775765084.851178:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765084.851181:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775765084.851459:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765084.851461:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775765084.851742:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765084.851746:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775765084.852045:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765084.852047:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775765089.969873:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765089.969879:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775765089.970509:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765089.970511:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775765089.970707:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765089.970709:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775765089.970950:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765089.970959:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775765089.971261:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765089.971263:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775765095.089873:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765095.089879:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775765095.090482:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765095.090484:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775765095.090674:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765095.090676:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775765095.090850:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765095.090852:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775765095.091100:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765095.091102:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775765100.209873:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765100.209880:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775765100.210505:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765100.210507:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775765100.210708:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765100.210710:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775765100.210920:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765100.210922:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775765100.211144:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765100.211145:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775765105.330914:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765105.331859:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765105.332228:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765105.332525:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765105.332777:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765110.451024:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765110.451037:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775765110.452353:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765110.452360:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775765110.453931:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765110.453950:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775765110.454662:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765110.454666:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775765110.454977:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765110.454979:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775765115.570414:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765115.570426:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1775765115.571583:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765115.571586:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1775765115.571970:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765115.571973:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1775765115.572388:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765115.572392:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1775765115.572791:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765115.572794:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1775765120.689885:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765120.689905:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1775765120.690522:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765120.690525:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1775765120.690732:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765120.690734:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1775765120.690968:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765120.690971:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1775765120.691212:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765120.691214:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1775765125.810308:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765125.810317:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1775765125.811514:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765125.811516:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1775765125.811950:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765125.811992:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1775765125.812403:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765125.812405:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1775765125.812778:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765125.812783:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1775765130.929916:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765130.929922:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775765130.930913:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765130.930915:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775765130.931298:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765130.931301:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775765130.931646:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765130.931649:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775765130.932082:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765130.932085:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775765136.050661:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765136.050676:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775765136.052005:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765136.052008:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775765136.052349:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765136.052352:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775765136.052647:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765136.052649:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775765136.053078:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765136.053101:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775765141.170637:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765141.170647:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775765141.172403:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765141.172406:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775765141.173193:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765141.173197:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775765141.173837:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765141.173839:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775765141.174279:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765141.174281:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775765146.289880:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765146.289886:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775765146.290528:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765146.290533:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775765146.290955:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765146.290957:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775765146.291196:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765146.291198:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775765146.291371:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765146.291373:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775765151.409776:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765151.409791:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775765151.410434:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765151.410435:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775765151.410716:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765151.410718:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775765151.410929:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765151.410931:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775765151.411357:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765151.411360:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775765156.530032:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765156.530038:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1775765156.530663:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765156.530665:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1775765156.530902:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765156.530904:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1775765156.531097:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765156.531099:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1775765156.531450:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765156.531453:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1775765161.650569:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765161.650580:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1775765161.652092:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765161.652095:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1775765161.652533:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765161.652536:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1775765161.653087:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765161.653090:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1775765161.653510:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765161.653513:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1775765166.770140:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765166.770147:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775765166.770865:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765166.770867:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775765166.771099:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765166.771101:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775765166.771311:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765166.771313:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775765166.771622:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765166.771625:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775765171.890282:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765171.890921:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765171.891426:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765171.891867:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765171.892301:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765177.010020:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765177.010028:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775765177.011001:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765177.011004:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775765177.011384:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765177.011386:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775765177.011655:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765177.011657:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775765177.011989:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765177.012008:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775765182.129774:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765182.129780:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775765182.130455:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765182.130457:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775765182.130700:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765182.130701:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775765182.130909:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765182.130910:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775765182.131213:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765182.131214:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775765187.250101:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765187.250109:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775765187.250775:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765187.250778:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775765187.251143:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765187.251146:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775765187.251571:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765187.251575:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775765187.251792:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765187.251794:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775765192.370699:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765192.370718:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775765192.372450:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765192.372481:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775765192.373230:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765192.373234:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775765192.373718:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765192.373721:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775765192.374065:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765192.374068:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775765197.489963:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765197.489971:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775765197.490720:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765197.490723:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775765197.491030:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765197.491033:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775765197.491404:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765197.491407:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775765197.491696:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765197.491699:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775765202.610414:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765202.610425:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775765202.611846:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765202.611853:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775765202.612590:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765202.612594:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775765202.613233:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765202.613246:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775765202.613482:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765202.613485:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775765207.729764:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765207.729767:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775765207.730477:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765207.730479:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775765207.730720:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765207.730721:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775765207.730962:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765207.730964:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775765207.731219:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765207.731221:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775765212.850329:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765212.850340:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775765212.851140:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765212.851143:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775765212.851628:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765212.851632:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775765212.852039:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765212.852043:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775765212.852371:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765212.852374:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775765217.970211:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765217.970236:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775765217.971052:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765217.971055:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775765217.971376:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765217.971378:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775765217.971665:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765217.971667:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775765217.972075:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765217.972079:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775765223.090091:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765223.090096:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775765223.090879:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765223.090884:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775765223.091301:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765223.091304:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775765223.091570:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765223.091572:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775765223.091889:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765223.091892:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775765228.210442:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765228.210450:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775765228.211855:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765228.211863:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775765228.212491:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765228.212493:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775765228.212783:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765228.212785:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775765228.212986:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765228.212988:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775765233.329953:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765233.329961:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775765233.330919:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765233.330924:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775765233.331543:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765233.331546:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775765233.332200:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765233.332203:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775765233.332642:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765233.332645:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775765238.449868:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765238.450177:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765238.450523:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765238.450832:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765238.451019:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765243.570513:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765243.570523:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775765243.571654:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765243.571658:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775765243.572048:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765243.572051:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775765243.572406:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765243.572410:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775765243.572808:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765243.572811:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775765248.690071:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765248.690078:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775765248.691260:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765248.691262:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775765248.691508:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765248.691511:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775765248.691727:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765248.691729:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775765248.692127:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765248.692129:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775765253.809908:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765253.809915:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775765253.810669:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765253.810672:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775765253.810976:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765253.810977:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775765253.811213:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765253.811214:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775765253.811517:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765253.811519:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775765258.931251:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765258.931259:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775765258.932755:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765258.932758:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775765258.933202:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765258.933205:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775765258.933621:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765258.933625:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775765258.934022:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765258.934025:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775765264.051263:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765264.051283:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775765264.053802:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765264.053810:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775765264.055307:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765264.055317:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775765264.056812:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765264.056815:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775765264.057247:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765264.057250:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775765269.170120:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765269.170133:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1775765269.171108:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765269.171111:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1775765269.171455:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765269.171458:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1775765269.171672:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765269.171675:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1775765269.171917:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765269.171919:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1775765274.290381:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765274.290390:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1775765274.291530:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765274.291533:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1775765274.291925:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765274.291928:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1775765274.292304:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765274.292307:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1775765274.292742:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765274.292745:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1775765279.410492:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765279.410505:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 23 secs later) 00000800:00000200:2.1:1775765279.411865:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765279.411868:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 23 secs later) 00000800:00000200:2.1:1775765279.412370:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765279.412373:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 23 secs later) 00000800:00000200:2.1:1775765279.413303:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765279.413306:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 23 secs later) 00000800:00000200:2.1:1775765279.413658:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765279.413660:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 23 secs later) 00000800:00000200:2.1:1775765284.530132:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765284.530137:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775765284.530935:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765284.530938:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775765284.531208:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765284.531211:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775765284.531408:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765284.531410:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775765284.531628:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765284.531630:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775765289.650598:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765289.650606:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775765289.651718:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765289.651720:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775765289.651974:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765289.651985:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775765289.652302:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765289.652304:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775765289.652646:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765289.652648:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775765294.770513:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765294.770521:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775765294.772046:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765294.772055:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775765294.773445:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765294.773464:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775765294.774535:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765294.774537:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775765294.774744:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765294.774746:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775765299.890324:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765299.890331:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775765299.891533:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765299.891535:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775765299.891904:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765299.891909:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775765299.892328:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765299.892332:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775765299.892845:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765299.892849:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775765305.010162:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765305.010936:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765305.011317:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765305.011507:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765305.011682:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765310.130510:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765310.130520:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775765310.131700:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765310.131702:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775765310.131978:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765310.131998:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775765310.132267:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765310.132270:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775765310.132719:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765310.132722:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775765315.249807:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765315.249812:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775765315.250501:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765315.250506:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775765315.250781:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765315.250783:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775765315.251060:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765315.251062:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775765315.251259:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765315.251274:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775765320.370441:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765320.370449:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775765320.371675:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765320.371679:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775765320.372241:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765320.372246:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775765320.372736:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765320.372740:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775765320.373098:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765320.373101:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775765325.490704:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765325.490714:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775765325.492494:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765325.492502:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775765325.493564:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765325.493569:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775765325.494638:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765325.494644:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775765325.495226:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765325.495229:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775765330.609892:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765330.609897:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775765330.610554:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765330.610556:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775765330.610745:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765330.610747:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775765330.610917:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765330.610919:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775765330.611190:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765330.611192:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775765335.730448:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765335.730460:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775765335.731675:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765335.731678:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775765335.732039:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765335.732042:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775765335.732452:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765335.732455:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775765335.732808:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765335.732812:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000080:00000010:3.1:1775765335.801233:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472f00. 00000080:00000010:3.1:1775765335.801256:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9942074725c0. 00000080:00000010:3.1:1775765335.801261:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207472940. 00010000:00000010:3.1:1775765335.805233:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793b400. 00010000:00000010:3.1:1775765335.805244:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c400. 00010000:00000010:3.1:1775765335.805248:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079da400. 00010000:00000010:3.1:1775765335.805253:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0800. 00010000:00000010:3.1:1775765335.805257:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420694c600. 00010000:00000010:3.1:1775765335.805260:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb9400. 00010000:00000010:3.1:1775765335.805264:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb8c00. 00010000:00000010:3.1:1775765335.805267:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb3c00. 00010000:00000010:3.1:1775765335.805271:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2000. 00010000:00000010:3.1:1775765335.805274:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cbde00. 00010000:00000010:3.1:1775765335.805278:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1200. 00010000:00000010:3.1:1775765335.805281:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075daa00. 00010000:00000010:3.1:1775765335.805285:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb9200. 00010000:00000010:3.1:1775765335.805289:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420385ea00. 00010000:00000010:3.1:1775765335.805295:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075cb400. 00010000:00000010:3.1:1775765335.805299:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420385e400. 00010000:00000010:3.1:1775765335.805302:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075ca000. 00010000:00000010:3.1:1775765335.805306:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99421263af00. 00010000:00000010:3.1:1775765335.805310:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942420c0000. 00000800:00000200:2.1:1775765340.850001:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765340.850009:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775765340.850735:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765340.850738:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775765340.850953:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765340.850955:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775765340.851265:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765340.851268:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775765340.851740:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765340.851742:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775765345.970223:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765345.970231:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775765345.971126:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765345.971128:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775765345.971394:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765345.971396:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775765345.971678:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765345.971680:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775765345.972131:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765345.972133:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775765351.090540:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765351.090556:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775765351.091914:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765351.091916:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775765351.092290:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765351.092293:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775765351.092560:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765351.092562:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775765351.092991:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765351.093016:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775765356.210198:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765356.210212:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775765356.211286:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765356.211290:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775765356.211870:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765356.211874:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775765356.212538:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765356.212542:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775765356.213042:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765356.213043:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775765361.329975:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765361.329985:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775765361.331122:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765361.331124:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775765361.331370:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765361.331372:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775765361.331562:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765361.331563:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775765361.331816:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765361.331817:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00010000:00000010:1.1:1775765365.498218:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903600. 00010000:00000010:1.1:1775765365.498246:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1600. 00010000:00000010:1.1:1775765365.498251:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994206989e00. 00010000:00000010:1.1:1775765365.498256:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898000. 00010000:00000010:1.1:1775765365.498296:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7000. 00010000:00000010:1.1:1775765365.498300:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798da00. 00010000:00000010:1.1:1775765365.498304:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078add00. 00010000:00000010:1.1:1775765365.498308:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798ca00. 00010000:00000010:1.1:1775765365.498311:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ad200. 00010000:00000010:1.1:1775765365.498314:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207534800. 00010000:00000010:1.1:1775765365.502235:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207926600. 00010000:00000010:1.1:1775765365.502245:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207938800. 00010000:00000010:1.1:1775765365.502249:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207926500. 00010000:00000010:1.1:1775765365.502252:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789d200. 00010000:00000010:1.1:1775765365.502256:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079cb200. 00010000:00000010:1.1:1775765365.502260:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079cb600. 00010000:00000010:1.1:1775765365.502263:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079ca600. 00010000:00000010:1.1:1775765365.502266:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942079cba00. 00010000:00000010:1.1:1775765365.502286:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898a00. 00010000:00000010:1.1:1775765365.502289:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793a400. 00010000:00000010:1.1:1775765365.502292:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894e00. 00000800:00000200:2.1:1775765366.450626:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765366.450636:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775765366.451926:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765366.451929:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775765366.453002:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765366.453013:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775765366.453614:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765366.453616:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775765366.453814:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765366.453815:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775765371.569836:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765371.570260:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765371.570515:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765371.570684:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765371.570844:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765376.689913:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765376.689918:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775765376.690655:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765376.690658:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775765376.690949:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765376.690951:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775765376.691133:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765376.691135:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775765376.691555:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765376.691558:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775765381.810546:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765381.810558:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775765381.811952:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765381.811955:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775765381.812351:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765381.812354:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775765381.812719:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765381.812722:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775765381.813234:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765381.813237:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775765386.930245:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765386.930256:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775765386.931779:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765386.931783:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775765386.932719:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765386.932725:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775765386.933710:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765386.933719:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775765386.934075:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765386.934077:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775765392.049789:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765392.049795:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775765392.050332:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765392.050334:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775765392.050513:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765392.050515:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775765392.050726:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765392.050728:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775765392.050900:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765392.050902:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775765397.170219:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765397.170225:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775765397.171322:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765397.171324:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775765397.171570:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765397.171572:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775765397.171765:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765397.171767:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775765397.172019:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765397.172021:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775765402.289995:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765402.289999:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1775765402.290665:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765402.290666:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1775765402.290877:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765402.290879:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1775765402.291142:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765402.291144:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1775765402.291420:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765402.291422:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000080:00000010:3.1:1775765406.455186:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6bca0. 00000080:00000010:3.1:1775765406.455202:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b780. 00000080:00000010:3.1:1775765406.459185:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a460. 00010000:00000010:3.1:1775765406.459236:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207976b00. 00010000:00000010:3.1:1775765406.459254:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895600. 00010000:00000010:3.1:1775765406.459257:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994211bdb500. 00010000:00000010:3.1:1775765406.459260:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789cc00. 00010000:00000010:3.1:1775765406.459262:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207c60600. 00010000:00000010:3.1:1775765406.459264:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789c200. 00010000:00000010:3.1:1775765406.459266:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994211bdbd00. 00010000:00000010:3.1:1775765406.459268:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069a7600. 00010000:00000010:3.1:1775765406.459270:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994211bdb800. 00010000:00000010:3.1:1775765406.459272:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db0400. 00010000:00000010:3.1:1775765406.459274:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994211bdbc00. 00010000:00000010:3.1:1775765406.459276:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994203fb1200. 00010000:00000010:3.1:1775765406.459278:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994218fe6b00. 00010000:00000010:3.1:1775765406.459281:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd800. 00010000:00000010:3.1:1775765406.459283:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078a1700. 00010000:00000010:3.1:1775765406.459285:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd400. 00010000:00000010:3.1:1775765406.459287:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994211bdb200. 00010000:00000010:3.1:1775765406.459289:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc200. 00010000:00000010:3.1:1775765406.459291:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994219da2300. 00010000:00000010:3.1:1775765406.459293:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917400. 00010000:00000010:3.1:1775765406.459295:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078a1e00. 00010000:00000010:3.1:1775765406.459297:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798dc00. 00010000:00000010:3.1:1775765406.459299:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078e9d00. 00010000:00000010:3.1:1775765406.459301:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207db1800. 00010000:00000010:3.1:1775765406.459309:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078b3e00. 00010000:00000010:3.1:1775765406.459311:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb2400. 00010000:00000010:3.1:1775765406.459313:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994220375300. 00010000:00000010:3.1:1775765406.459315:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942420c0200. 00010000:00000010:3.1:1775765406.459318:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994220375b00. 00010000:00000010:3.1:1775765406.459319:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994209969e00. 00010000:00000010:3.1:1775765406.459322:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d89a00. 00000800:00000200:2.1:1775765407.410121:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765407.410131:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1775765407.411148:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765407.411152:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1775765407.411611:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765407.411615:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1775765407.411861:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765407.411863:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1775765407.412116:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765407.412118:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1775765412.530040:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765412.530047:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775765412.530732:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765412.530734:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775765412.530921:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765412.530922:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775765412.531100:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765412.531102:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775765412.531392:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765412.531393:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775765417.649940:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765417.649945:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775765417.650880:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765417.650887:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775765417.651477:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765417.651482:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775765417.651985:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765417.651990:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775765417.652418:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765417.652419:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775765422.769775:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765422.769781:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775765422.770493:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765422.770495:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775765422.770681:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765422.770683:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775765422.770898:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765422.770900:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775765422.771243:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765422.771245:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775765427.890324:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765427.890333:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775765427.891528:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765427.891535:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775765427.891938:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765427.891941:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775765427.892308:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765427.892311:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775765427.892805:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765427.892808:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775765433.010411:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765433.010429:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775765433.011652:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765433.011656:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775765433.012280:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765433.012283:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775765433.012741:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765433.012746:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775765433.013349:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765433.013352:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775765438.129923:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765438.130304:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765438.130544:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765438.130663:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765438.130807:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765443.251953:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765443.251967:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775765443.253017:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765443.253021:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775765443.253471:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765443.253473:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775765443.253858:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765443.253861:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775765443.254148:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765443.254151:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00010000:00000010:0.1:1775765446.893293:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942079b7b00. 00010000:00000010:0.1:1775765446.893305:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420694ce00. 00010000:00000010:0.1:1775765446.893315:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942079b7200. 00010000:00000010:0.1:1775765446.893332:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207939000. 00010000:00000010:0.1:1775765446.893343:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99421805eb00. 00010000:00000010:0.1:1775765446.893354:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994209969600. 00010000:00000010:0.1:1775765446.893358:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420385e700. 00010000:00000010:0.1:1775765446.893361:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994218f6ea00. 00010000:00000010:0.1:1775765446.893382:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99421263a500. 00010000:00000010:0.1:1775765446.893385:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075db400. 00010000:00000010:0.1:1775765446.893388:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420385e200. 00010000:00000010:0.1:1775765446.893390:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075db800. 00010000:00000010:0.1:1775765446.893392:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99421805e300. 00010000:00000010:0.1:1775765446.893394:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994201482800. 00010000:00000010:0.1:1775765446.893398:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209c33000. 00010000:00000010:0.1:1775765446.893401:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942013d3c00. 00010000:00000010:0.1:1775765446.893404:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209c33600. 00010000:00000010:0.1:1775765446.893406:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075dbc00. 00010000:00000010:0.1:1775765446.893409:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942079b7f00. 00010000:00000010:0.1:1775765446.893411:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793d600. 00010000:00000010:0.1:1775765446.893413:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99421805e500. 00010000:00000010:0.1:1775765446.893415:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793c800. 00010000:00000010:0.1:1775765446.893418:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994218fe6d00. 00010000:00000010:0.1:1775765446.893420:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420694d800. 00010000:00000010:0.1:1775765446.893422:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994218fe6e00. 00010000:00000010:0.1:1775765446.893424:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207939600. 00010000:00000010:0.1:1775765446.893427:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942069a5c00. 00010000:00000010:0.1:1775765446.893429:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ce0000. 00010000:00000010:0.1:1775765446.893431:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207c67d00. 00010000:00000010:0.1:1775765446.893433:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cbc000. 00010000:00000010:0.1:1775765446.893435:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942069a5000. 00010000:00000010:0.1:1775765446.893437:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cb9800. 00010000:00000010:3.1:1775765446.897222:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99421263a600. 00010000:00000010:3.1:1775765446.897227:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994218f6f400. 00010000:00000010:3.1:1775765446.897231:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420385e300. 00010000:00000010:3.1:1775765446.897233:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994203fb0c00. 00010000:00000010:3.1:1775765446.897235:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994220375400. 00010000:00000010:3.1:1775765446.897237:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994201483800. 00010000:00000010:3.1:1775765446.897239:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420385e600. 00010000:00000010:3.1:1775765446.897241:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994201483200. 00010000:00000010:3.1:1775765446.897242:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994220375e00. 00010000:00000010:3.1:1775765446.897244:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420b14b200. 00010000:00000010:3.1:1775765446.897246:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209c33700. 00010000:00000010:3.1:1775765446.897248:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942013d3600. 00010000:00000010:3.1:1775765446.897250:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209c33b00. 00010000:00000010:3.1:1775765446.897252:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942013d3a00. 00010000:00000010:3.1:1775765446.897254:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99421263a800. 00010000:00000010:3.1:1775765446.897256:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994218f6f200. 00010000:00000010:3.1:1775765446.897257:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994210a04000. 00010000:00000010:3.1:1775765446.897259:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994203fb1800. 00010000:00000010:3.1:1775765446.897261:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966100. 00010000:00000010:3.1:1775765446.897263:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902200. 00010000:00000010:3.1:1775765446.897265:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994219ec1200. 00010000:00000010:3.1:1775765446.897267:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207939400. 00010000:00000010:3.1:1775765446.897269:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994209897800. 00010000:00000010:3.1:1775765446.897271:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994203fb1400. 00010000:00000010:3.1:1775765446.897272:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994211cc1c00. 00010000:00000010:3.1:1775765446.897274:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994203fb1c00. 00010000:00000010:3.1:1775765446.897276:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942079b7700. 00010000:00000010:3.1:1775765446.897278:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207939a00. 00010000:00000010:3.1:1775765446.897280:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942079b7800. 00010000:00000010:3.1:1775765446.897281:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ca0800. 00010000:00000010:3.1:1775765446.897283:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420b307700. 00010000:00000010:3.1:1775765446.897285:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793d400. 00010000:00000010:3.1:1775765446.897287:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942079b7e00. 00010000:00000010:3.1:1775765446.897289:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420793de00. 00010000:00000010:3.1:1775765446.897290:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420b307c00. 00010000:00000010:3.1:1775765446.897292:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420694c800. 00010000:00000010:3.1:1775765446.897294:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994218fe6700. 00010000:00000010:3.1:1775765446.897295:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207938c00. 00010000:00000010:3.1:1775765446.897297:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942069a5e00. 00010000:00000010:3.1:1775765446.897299:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207ce1a00. 00010000:00000010:3.1:1775765446.897301:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207c70c00. 00010000:00000010:3.1:1775765446.897302:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cbd400. 00010000:00000010:3.1:1775765446.897304:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207c67100. 00010000:00000010:3.1:1775765446.897306:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075db000. 00000800:00000200:2.1:1775765448.370027:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765448.370035:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775765448.371026:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765448.371033:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775765448.371799:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765448.371806:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775765448.372224:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765448.372227:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775765448.372513:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765448.372516:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775765453.489942:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765453.489948:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1775765453.490627:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765453.490629:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1775765453.490816:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765453.490817:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1775765453.491028:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765453.491030:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1775765453.491233:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765453.491235:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1775765458.610504:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765458.610516:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775765458.611711:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765458.611714:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775765458.612112:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765458.612115:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775765458.612562:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765458.612565:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775765458.612947:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765458.612950:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1775765463.730059:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765463.730066:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775765463.730910:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765463.730912:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775765463.731209:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765463.731211:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775765463.731520:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765463.731523:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775765463.731986:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765463.731988:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00010000:00000010:1.1:1775765465.851228:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207c70800. 00010000:00000010:1.1:1775765465.851238:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0400. 00010000:00000010:1.1:1775765465.851242:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207c70f00. 00010000:00000010:1.1:1775765465.851245:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd0e00. 00010000:00000010:1.1:1775765465.851248:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207eef600. 00010000:00000010:1.1:1775765465.851252:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898800. 00010000:00000010:1.1:1775765465.851255:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207c81400. 00010000:00000010:1.1:1775765465.851259:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0a00. 00010000:00000010:1.1:1775765465.851262:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207c81b00. 00010000:00000010:1.1:1775765465.851265:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0000. 00010000:00000010:1.1:1775765465.851269:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207c81a00. 00010000:00000010:1.1:1775765465.851272:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207cd1800. 00010000:00000010:1.1:1775765465.851276:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994218fe6400. 00010000:00000010:1.1:1775765465.851279:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898e00. 00010000:00000010:1.1:1775765465.851282:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207d0be00. 00010000:00000010:1.1:1775765465.851285:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420798c800. 00010000:00000010:1.1:1775765465.851303:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943b00. 00010000:00000010:1.1:1775765465.851307:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0c00. 00010000:00000010:1.1:1775765465.851310:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994220010500. 00010000:00000010:1.1:1775765465.851331:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7e00. 00010000:00000010:1.1:1775765465.851352:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207931000. 00010000:00000010:1.1:1775765465.851356:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80e00. 00010000:00000010:1.1:1775765465.851359:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420387cd00. 00010000:00000010:1.1:1775765465.851363:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970000. 00010000:00000010:1.1:1775765465.855195:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994218766e00. 00010000:00000010:1.1:1775765465.855203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fca00. 00010000:00000010:1.1:1775765465.855206:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078e9700. 00010000:00000010:1.1:1775765465.855209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207895200. 00000800:00000200:1.1:1775765468.850299:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765468.850324:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775765468.851519:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765468.851533:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775765468.852033:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765468.852037:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775765468.852471:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765468.852475:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775765468.852929:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765468.852933:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775765473.970496:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765473.970513:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775765473.971920:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765473.971922:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775765473.972152:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765473.972175:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775765473.972382:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765473.972385:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775765473.972671:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765473.972673:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775765479.089914:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765479.089919:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775765479.090872:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765479.090877:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775765479.091483:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765479.091488:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775765479.092110:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765479.092128:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775765479.093104:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765479.093109:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775765484.210419:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765484.210428:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775765484.211614:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765484.211617:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775765484.212025:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765484.212028:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775765484.212406:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765484.212409:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775765484.212934:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765484.212938:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00010000:00000010:0.1:1775765486.330203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0800. 00010000:00000010:0.1:1775765486.330210:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420788ee00. 00010000:00000010:0.1:1775765486.330213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789c800. 00010000:00000010:0.1:1775765486.330233:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917c00. 00010000:00000010:0.1:1775765486.330236:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942079a4400. 00010000:00000010:0.1:1775765486.330241:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789c400. 00010000:00000010:0.1:1775765486.330244:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899000. 00010000:00000010:0.1:1775765486.330248:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207894400. 00010000:00000010:0.1:1775765486.330266:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fda00. 00010000:00000010:0.1:1775765486.330271:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943600. 00010000:00000010:0.1:1775765486.330275:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1800. 00010000:00000010:0.1:1775765486.334191:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943900. 00010000:00000010:0.1:1775765486.334196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e0600. 00010000:00000010:0.1:1775765486.334199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899200. 00010000:00000010:0.1:1775765486.334201:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966800. 00010000:00000010:0.1:1775765486.334204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899a00. 00010000:00000010:0.1:1775765486.334206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e7600. 00010000:00000010:0.1:1775765486.334209:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207de4400. 00010000:00000010:0.1:1775765486.334211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942078e6800. 00010000:00000010:0.1:1775765486.334213:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207966f00. 00010000:00000010:0.1:1775765486.334216:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899800. 00010000:00000010:0.1:1775765486.334218:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207902a00. 00010000:00000010:0.1:1775765486.334220:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899c00. 00000800:00000200:1.1:1775765489.329867:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765489.329873:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775765489.330701:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765489.330703:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775765489.330996:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765489.330999:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775765489.331245:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765489.331247:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775765489.331670:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765489.331672:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775765494.450490:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765494.450512:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775765494.451363:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765494.451365:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775765494.451669:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765494.451671:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775765494.451895:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765494.451897:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775765494.452238:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765494.452241:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000080:00000010:3.1:1775765495.545217:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6aa00. 00000080:00000010:3.1:1775765495.549196:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b6a0. 00000080:00000010:3.1:1775765495.549202:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207db21a0. 00000080:00000010:3.1:1775765495.549204:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994218f6a080. 00000080:00000010:3.1:1775765495.549207:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff994207d6b300. 00010000:00000010:3.1:1775765495.549210:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207de4300. 00010000:00000010:3.1:1775765495.549213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207903800. 00010000:00000010:3.1:1775765495.549216:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943700. 00010000:00000010:3.1:1775765495.549218:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7000. 00010000:00000010:3.1:1775765495.549220:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916c00. 00010000:00000010:3.1:1775765495.549222:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81000. 00010000:00000010:3.1:1775765495.549224:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b7e00. 00010000:00000010:3.1:1775765495.549226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80c00. 00010000:00000010:3.1:1775765495.549227:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942069a5f00. 00010000:00000010:3.1:1775765495.549229:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81c00. 00010000:00000010:3.1:1775765495.549231:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207de4c00. 00010000:00000010:3.1:1775765495.549232:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207971000. 00010000:00000010:3.1:1775765495.549234:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078efe00. 00010000:00000010:3.1:1775765495.549236:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81a00. 00010000:00000010:3.1:1775765495.549237:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207de4a00. 00010000:00000010:3.1:1775765495.549239:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970c00. 00010000:00000010:3.1:1775765495.549241:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943800. 00010000:00000010:3.1:1775765495.549242:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207970800. 00010000:00000010:3.1:1775765495.549244:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ef500. 00010000:00000010:3.1:1775765495.549245:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d81600. 00010000:00000010:3.1:1775765495.549247:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207de4200. 00010000:00000010:3.1:1775765495.549248:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6e00. 00010000:00000010:3.1:1775765495.549250:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ef200. 00010000:00000010:3.1:1775765495.549251:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207d80a00. 00010000:00000010:3.1:1775765495.549253:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943200. 00010000:00000010:3.1:1775765495.549254:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6600. 00010000:00000010:3.1:1775765495.549256:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942069a5a00. 00010000:00000010:3.1:1775765495.549257:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994209db3000. 00010000:00000010:3.1:1775765495.549259:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207de4900. 00010000:00000010:3.1:1775765495.549261:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789da00. 00010000:00000010:3.1:1775765495.549263:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942069a5400. 00010000:00000010:3.1:1775765495.549265:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd000. 00010000:00000010:3.1:1775765495.549267:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207926200. 00010000:00000010:3.1:1775765495.549269:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fce00. 00010000:00000010:3.1:1775765495.549271:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943100. 00010000:00000010:3.1:1775765495.549273:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789dc00. 00010000:00000010:3.1:1775765495.549274:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994218fe6200. 00010000:00000010:3.1:1775765495.549276:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898600. 00010000:00000010:3.1:1775765495.549277:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943000. 00010000:00000010:3.1:1775765495.549279:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942069b6a00. 00010000:00000010:3.1:1775765495.549280:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ef100. 00010000:00000010:3.1:1775765495.549282:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc400. 00010000:00000010:3.1:1775765495.549283:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942079b7300. 00010000:00000010:3.1:1775765495.549285:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898200. 00010000:00000010:3.1:1775765495.549286:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ef000. 00010000:00000010:3.1:1775765495.549288:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fde00. 00010000:00000010:3.1:1775765495.549290:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942079b7100. 00010000:00000010:3.1:1775765495.549299:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207899e00. 00010000:00000010:3.1:1775765495.549301:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994211bdba00. 00010000:00000010:3.1:1775765495.549303:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fd600. 00010000:00000010:3.1:1775765495.549304:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207de4700. 00010000:00000010:3.1:1775765495.549306:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789ce00. 00010000:00000010:3.1:1775765495.549308:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420788e700. 00010000:00000010:3.1:1775765495.549309:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942075fc600. 00010000:00000010:3.1:1775765495.549311:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420788eb00. 00010000:00000010:3.1:1775765495.549312:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789d000. 00010000:00000010:3.1:1775765495.549314:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942079a4d00. 00010000:00000010:3.1:1775765495.549316:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207898400. 00010000:00000010:3.1:1775765495.549317:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789d400. 00010000:00000010:3.1:1775765495.549319:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff9942078ad000. 00010000:00000010:3.1:1775765495.549320:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789ca00. 00010000:00000010:3.1:1775765495.549322:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916200. 00010000:00000010:3.1:1775765495.549324:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207943f00. 00010000:00000010:3.1:1775765495.549326:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff99420789d800. 00010000:00000010:3.1:1775765495.549328:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207917a00. 00010000:00000010:3.1:1775765495.549329:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1400. 00010000:00000010:3.1:1775765495.549331:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994207de4000. 00010000:00000010:3.1:1775765495.549333:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9942182e1200. 00010000:00000010:3.1:1775765495.549334:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff994210404000. 00010000:00000010:3.1:1775765495.549336:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916000. 00010000:00000010:3.1:1775765495.549337:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff99420788e400. 00010000:00000010:3.1:1775765495.549339:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff994207916a00. 00000800:00000200:2.1:1775765499.570502:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765499.570509:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775765499.572120:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765499.572133:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775765499.572486:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765499.572488:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775765499.572861:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765499.572863:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775765499.573128:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765499.573130:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775765504.690753:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765504.691688:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765504.692014:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765504.692386:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765504.692695:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775765509.810368:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775765509.810375:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775765509.811715:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775765509.811721:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775765509.812359:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775765509.812362:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775765509.812586:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775765509.812588:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775765509.812890:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775765509.812893:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775765514.930433:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775765514.930440:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775765514.931802:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775765514.931804:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775765514.932159:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775765514.932161:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775765514.932489:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775765514.932492:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775765514.932839:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775765514.932842:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775765520.050510:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775765520.050520:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775765520.051546:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775765520.051549:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775765520.051904:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775765520.051907:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775765520.052413:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775765520.052415:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775765520.052655:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775765520.052657:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775765525.169894:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775765525.169898:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775765525.170576:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775765525.170578:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775765525.170778:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775765525.170780:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775765525.170978:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775765525.170980:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775765525.171304:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775765525.171306:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775765530.289893:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775765530.289900:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775765530.290620:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775765530.290622:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775765530.290840:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775765530.290842:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775765530.291157:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775765530.291160:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775765530.291544:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775765530.291546:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775765535.410855:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765535.410869:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775765535.412095:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765535.412099:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775765535.412626:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765535.412629:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775765535.413002:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765535.413007:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775765535.413537:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765535.413540:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1775765540.531203:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765540.531210:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775765540.533859:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765540.533870:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775765540.535373:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765540.535385:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775765540.536986:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765540.536990:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775765540.537559:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765540.537562:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775765545.650645:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765545.650660:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775765545.652151:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765545.652153:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775765545.652390:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765545.652392:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775765545.652623:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765545.652625:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775765545.653007:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765545.653010:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775765550.770792:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765550.770798:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775765550.772454:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765550.772456:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775765550.772824:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765550.772827:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775765550.773143:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765550.773146:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775765550.773647:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765550.773651:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775765555.890857:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765555.890868:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775765555.892675:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765555.892679:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775765555.893071:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765555.893074:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775765555.893453:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765555.893457:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775765555.893923:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765555.893926:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775765561.009883:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765561.009888:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775765561.010703:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765561.010705:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775765561.010996:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765561.010999:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775765561.011249:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765561.011252:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775765561.011596:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765561.011598:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775765566.130297:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765566.130308:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775765566.131239:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765566.131242:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775765566.131511:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765566.131513:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775765566.131770:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765566.131773:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775765566.132058:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765566.132061:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775765571.249914:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765571.250340:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765571.250770:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765571.250931:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765571.251106:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765576.370362:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765576.370366:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775765576.370740:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765576.370742:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775765576.371212:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765576.371218:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775765576.371834:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765576.371837:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775765576.372134:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765576.372136:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775765581.490149:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765581.490196:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1775765581.491388:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765581.491390:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1775765581.491649:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765581.491652:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1775765581.491933:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765581.491935:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1775765581.492279:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765581.492281:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1775765586.610254:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765586.610260:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1775765586.611236:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765586.611242:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1775765586.611544:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765586.611563:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1775765586.612017:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765586.612020:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1775765586.612430:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765586.612432:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1775765591.729912:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765591.729934:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1775765591.730556:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765591.730558:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1775765591.730782:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765591.730784:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1775765591.730993:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765591.730994:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1775765591.731255:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765591.731257:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775765596.850124:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765596.850132:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775765596.850971:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765596.850974:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775765596.851355:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765596.851358:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775765596.851753:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765596.851757:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775765596.852088:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765596.852091:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1775765601.971078:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765601.971100:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775765601.973575:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765601.973603:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775765601.974776:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765601.974783:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775765601.975602:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765601.975605:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775765601.976035:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765601.976040:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775765607.090118:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765607.090131:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775765607.091175:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765607.091179:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775765607.091681:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765607.091684:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775765607.092075:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765607.092077:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775765607.092618:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765607.092622:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775765612.210231:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765612.210237:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775765612.211063:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765612.211065:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775765612.211381:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765612.211384:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775765612.211674:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765612.211677:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775765612.212028:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765612.212030:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775765617.330247:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765617.330254:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775765617.331379:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765617.331382:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775765617.331734:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765617.331737:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775765617.332047:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765617.332050:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775765617.332546:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765617.332549:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775765622.449845:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765622.449849:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775765622.450519:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765622.450521:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775765622.450772:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765622.450773:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775765622.450969:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765622.450971:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775765622.451316:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765622.451318:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775765627.571119:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765627.571131:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1775765627.572510:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765627.572513:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1775765627.572873:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765627.572875:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1775765627.573318:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765627.573322:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1775765627.573792:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765627.573795:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1775765632.690480:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765632.690494:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1775765632.691963:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765632.691967:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1775765632.692853:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765632.692858:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1775765632.693547:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765632.693550:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1775765632.693976:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765632.693979:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1775765637.810618:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765637.811439:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765637.812033:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765637.812438:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765637.812732:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765642.929932:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765642.929938:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775765642.930747:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765642.930749:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775765642.931021:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765642.931023:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775765642.931373:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765642.931375:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775765642.931789:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765642.931792:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775765648.049904:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765648.049909:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775765648.050533:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765648.050535:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775765648.050788:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765648.050791:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775765648.051016:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765648.051020:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775765648.051320:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765648.051322:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775765653.170404:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765653.170414:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775765653.171604:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765653.171608:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775765653.171987:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765653.171990:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775765653.172483:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765653.172486:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775765653.172871:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765653.172874:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775765658.290105:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765658.290112:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775765658.290749:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765658.290752:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775765658.291219:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765658.291222:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775765658.291588:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765658.291591:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775765658.292128:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765658.292130:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775765663.411191:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765663.411200:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775765663.413653:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765663.413667:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775765663.414885:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765663.414890:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775765663.415492:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765663.415497:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775765663.415984:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765663.415988:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775765668.530560:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765668.530571:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775765668.531542:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765668.531545:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775765668.531982:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765668.532002:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775765668.532408:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765668.532411:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775765668.532753:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765668.532756:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775765673.649737:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765673.649743:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775765673.650440:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765673.650442:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775765673.650634:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765673.650636:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775765673.650875:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765673.650877:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775765673.651194:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765673.651196:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775765678.769670:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765678.769682:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775765678.770101:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765678.770103:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775765678.770313:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765678.770315:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775765678.770492:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765678.770494:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775765678.770729:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765678.770731:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775765683.889726:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765683.889731:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775765683.890351:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765683.890356:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775765683.890853:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765683.890855:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775765683.891038:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765683.891040:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775765683.891268:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765683.891270:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775765689.010011:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765689.010018:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775765689.010789:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765689.010790:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775765689.010993:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765689.010995:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775765689.011206:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765689.011208:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775765689.011450:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765689.011452:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775765694.129934:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765694.129941:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775765694.130682:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765694.130685:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775765694.131010:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765694.131013:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775765694.131343:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765694.131345:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775765694.131605:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765694.131607:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775765699.249973:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765699.249977:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775765699.250996:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765699.251003:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775765699.251487:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765699.251489:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775765699.251771:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765699.251774:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775765699.251968:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765699.251970:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775765704.370195:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765704.370692:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765704.370859:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765704.371027:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765704.371227:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765709.490006:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765709.490012:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 57 secs later) 00000800:00000200:2.1:1775765709.490739:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765709.490741:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 57 secs later) 00000800:00000200:2.1:1775765709.491026:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765709.491027:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 57 secs later) 00000800:00000200:2.1:1775765709.491307:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765709.491309:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 57 secs later) 00000800:00000200:2.1:1775765709.491588:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765709.491591:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 57 secs later) 00000800:00000200:2.1:1775765714.609890:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765714.609898:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1775765714.610694:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765714.610697:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1775765714.611037:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765714.611041:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1775765714.611399:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765714.611402:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1775765714.611786:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765714.611790:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1775765719.730552:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765719.730558:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1775765719.731888:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765719.731891:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1775765719.732229:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765719.732232:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1775765719.732583:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765719.732586:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1775765719.732991:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765719.732994:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1775765724.850038:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765724.850045:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1775765724.850970:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765724.850974:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1775765724.851571:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765724.851575:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1775765724.851783:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765724.851785:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1775765724.852036:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765724.852038:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1775765729.970353:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765729.970364:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1775765729.971485:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765729.971488:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1775765729.971870:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765729.971874:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1775765729.972375:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765729.972380:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1775765729.972734:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765729.972738:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1775765735.089918:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765735.089924:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775765735.090627:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765735.090629:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775765735.090886:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765735.090888:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775765735.091169:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765735.091171:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775765735.091470:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765735.091472:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775765740.210198:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765740.210202:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775765740.211512:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765740.211516:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775765740.211850:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765740.211854:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775765740.212202:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765740.212205:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775765740.212695:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765740.212698:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775765745.330311:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765745.330317:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775765745.331240:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765745.331243:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775765745.331525:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765745.331528:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775765745.331834:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765745.331837:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775765745.332395:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765745.332398:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775765750.450245:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765750.450252:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775765750.451087:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765750.451089:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775765750.451515:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765750.451517:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775765750.451797:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765750.451799:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775765750.452080:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765750.452083:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775765755.570211:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765755.570216:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1775765755.571125:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765755.571131:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1775765755.571639:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765755.571642:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1775765755.572227:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765755.572229:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1775765755.572607:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765755.572609:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1775765760.690084:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765760.690093:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1775765760.690927:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765760.690929:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1775765760.691184:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765760.691186:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1775765760.691379:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765760.691380:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1775765760.691600:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765760.691602:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1775765765.809936:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765765.809942:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775765765.810717:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765765.810725:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775765765.811194:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765765.811196:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775765765.811586:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765765.811588:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775765765.811763:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765765.811765:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775765770.929737:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765770.930079:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765770.930431:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765770.930667:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765770.930822:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765776.049792:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765776.049798:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775765776.050403:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765776.050406:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775765776.050752:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765776.050755:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775765776.051100:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765776.051103:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775765776.051432:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765776.051435:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775765781.170300:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765781.170311:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775765781.171408:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765781.171409:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775765781.171666:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765781.171668:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775765781.171905:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765781.171907:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775765781.172178:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765781.172180:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775765786.289910:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765786.289917:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775765786.290760:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765786.290763:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775765786.291410:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765786.291415:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775765786.291812:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765786.291814:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775765786.292003:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765786.292006:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775765791.409911:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765791.409917:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775765791.410531:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765791.410533:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775765791.410808:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765791.410809:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775765791.410995:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765791.410997:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775765791.411317:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765791.411330:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775765796.529784:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765796.529787:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775765796.530364:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765796.530367:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775765796.530629:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765796.530631:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775765796.530847:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765796.530848:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775765796.531231:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765796.531233:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1775765801.649829:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765801.649834:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775765801.650552:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765801.650554:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775765801.650785:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765801.650786:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775765801.650994:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765801.650996:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775765801.651305:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765801.651308:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775765806.770449:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765806.770463:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775765806.771242:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765806.771246:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775765806.771666:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765806.771670:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775765806.772078:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765806.772082:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775765806.772434:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765806.772436:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775765811.890029:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765811.890036:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775765811.890943:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765811.890945:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775765811.891278:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765811.891280:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775765811.891496:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765811.891498:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775765811.891873:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765811.891875:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775765817.010148:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765817.010177:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775765817.011439:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765817.011446:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775765817.011917:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765817.011918:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775765817.012134:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765817.012136:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775765817.012413:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765817.012415:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775765822.131256:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765822.131261:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775765822.132185:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765822.132187:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775765822.132410:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765822.132412:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775765822.132634:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765822.132636:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775765822.132845:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765822.132847:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775765827.249816:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765827.249820:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775765827.250321:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765827.250324:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775765827.250586:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765827.250588:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775765827.250796:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765827.250798:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775765827.251110:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765827.251112:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775765832.370124:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765832.370131:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775765832.372234:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765832.372237:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775765832.372517:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765832.372519:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775765832.372754:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765832.372756:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775765832.373053:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765832.373055:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775765837.489903:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765837.490354:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765837.490589:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765837.490768:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765837.490939:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765842.611039:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765842.611043:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775765842.611485:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765842.611488:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775765842.611850:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765842.611853:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775765842.612233:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765842.612236:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775765842.612581:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765842.612584:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775765847.730074:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765847.730081:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775765847.731440:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765847.731446:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775765847.732372:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765847.732375:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775765847.732657:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765847.732659:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775765847.734240:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765847.734242:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775765852.850564:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765852.850572:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775765852.851757:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765852.851760:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775765852.852107:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765852.852111:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775765852.852523:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765852.852526:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775765852.852986:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765852.852990:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775765857.969875:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765857.969879:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775765857.970482:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765857.970484:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775765857.970684:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765857.970686:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775765857.970875:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765857.970877:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775765857.971056:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765857.971058:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775765863.090144:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765863.090194:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775765863.091441:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765863.091444:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775765863.091740:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765863.091742:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775765863.091956:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765863.091958:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775765863.092334:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765863.092336:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775765868.210215:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765868.210224:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1775765868.210936:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765868.210939:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1775765868.211299:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765868.211302:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1775765868.211614:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765868.211617:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1775765868.211930:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765868.211933:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775765873.330347:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765873.330357:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1775765873.331253:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765873.331255:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1775765873.331516:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765873.331518:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1775765873.331747:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765873.331749:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1775765873.332105:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765873.332109:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1775765878.450297:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765878.450302:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 23 secs later) 00000800:00000200:1.1:1775765878.451763:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765878.451771:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 23 secs later) 00000800:00000200:1.1:1775765878.452747:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765878.452753:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 23 secs later) 00000800:00000200:1.1:1775765878.453236:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765878.453238:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 23 secs later) 00000800:00000200:1.1:1775765878.453496:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765878.453498:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 23 secs later) 00000800:00000200:0.1:1775765883.569894:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765883.569899:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775765883.570564:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765883.570569:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775765883.570897:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765883.570899:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775765883.571180:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765883.571182:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775765883.571384:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765883.571386:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775765888.689911:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765888.689916:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775765888.690665:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765888.690667:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775765888.690909:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765888.690911:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775765888.691201:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765888.691203:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775765888.691605:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765888.691607:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775765893.810180:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765893.810185:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775765893.811020:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765893.811023:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775765893.811358:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765893.811361:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775765893.811666:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765893.811669:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775765893.811991:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765893.811994:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775765898.929988:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765898.929991:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775765898.930735:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765898.930737:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775765898.932180:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765898.932182:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775765898.932403:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765898.932405:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775765898.932568:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765898.932570:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775765904.050125:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765904.050468:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765904.050856:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765904.051040:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765904.051217:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765909.171080:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765909.171088:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775765909.171909:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765909.171934:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775765909.172306:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765909.172309:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775765909.172666:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765909.172668:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775765909.172907:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765909.172909:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775765914.290021:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765914.290026:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775765914.290536:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765914.290538:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775765914.290807:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765914.290809:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775765914.291084:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765914.291086:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775765914.291317:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765914.291319:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775765919.409890:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765919.409910:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775765919.410491:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765919.410494:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775765919.410874:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765919.410877:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775765919.411184:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765919.411204:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775765919.411579:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765919.411582:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775765924.529802:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765924.529809:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1775765924.530386:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765924.530389:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1775765924.530670:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765924.530672:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1775765924.530895:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765924.530913:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1775765924.531094:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765924.531096:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1775765929.649969:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765929.649980:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1775765929.650828:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765929.650831:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1775765929.651167:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765929.651184:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1775765929.651544:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765929.651546:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1775765929.651793:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765929.651796:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775765934.770648:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765934.770657:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775765934.771926:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765934.771929:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775765934.772297:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765934.772301:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775765934.772687:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765934.772690:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775765934.773180:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765934.773184:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775765939.889897:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765939.889906:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775765939.890831:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765939.890834:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775765939.891418:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765939.891423:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775765939.891855:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765939.891857:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775765939.892187:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765939.892188:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775765945.009827:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765945.009846:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775765945.010412:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765945.010414:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775765945.010597:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765945.010599:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775765945.010771:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765945.010773:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775765945.010943:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765945.010944:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775765950.129870:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765950.129875:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775765950.130407:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765950.130409:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775765950.130600:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765950.130602:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775765950.130788:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765950.130790:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775765950.130980:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765950.130982:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775765955.249850:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765955.249854:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775765955.250364:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765955.250367:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775765955.250612:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765955.250614:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775765955.250889:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765955.250891:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775765955.251212:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765955.251215:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775765960.369990:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765960.369996:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775765960.370589:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765960.370591:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775765960.370824:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765960.370826:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775765960.371012:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765960.371014:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775765960.371277:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775765960.371279:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775765965.490439:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765965.490450:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775765965.491768:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765965.491770:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775765965.491989:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765965.491990:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775765965.492327:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765965.492329:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775765965.492536:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765965.492538:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775765970.610432:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765970.611315:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765970.612063:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765970.612516:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765970.613029:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765975.730582:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765975.730593:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775765975.732105:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765975.732117:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775765975.732830:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765975.732833:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775765975.733290:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765975.733293:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775765975.733802:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765975.733805:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775765980.850280:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765980.850289:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775765980.851192:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765980.851196:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775765980.851467:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765980.851469:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775765980.851703:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765980.851706:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775765980.852042:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765980.852045:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775765985.969843:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765985.969852:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775765985.970510:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765985.970513:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775765985.970812:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765985.970815:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775765985.971118:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765985.971121:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775765985.971474:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775765985.971477:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775765991.090047:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765991.090054:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775765991.090951:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765991.090957:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775765991.091158:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765991.091191:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775765991.091458:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765991.091459:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775765991.091653:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765991.091654:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775765996.209935:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765996.209939:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775765996.210428:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765996.210429:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775765996.210623:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765996.210625:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775765996.210816:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765996.210818:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775765996.211060:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775765996.211062:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775766001.330427:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766001.330440:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1775766001.332142:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766001.332149:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1775766001.332552:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766001.332554:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1775766001.332865:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766001.332866:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1775766001.333110:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766001.333128:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1775766006.450398:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766006.450409:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1775766006.450751:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766006.450753:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1775766006.450941:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766006.450943:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1775766006.451179:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766006.451182:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1775766006.451450:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766006.451452:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1775766011.570129:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766011.570135:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775766011.571139:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766011.571141:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775766011.571400:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766011.571402:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775766011.571582:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766011.571584:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775766011.571853:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766011.571855:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775766016.689974:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766016.689980:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775766016.690683:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766016.690685:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775766016.690955:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766016.690957:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775766016.691191:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766016.691192:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775766016.691442:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766016.691444:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775766021.809882:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766021.809887:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775766021.810586:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766021.810589:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775766021.810864:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766021.810865:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775766021.811108:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766021.811110:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775766021.811389:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766021.811391:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775766026.929972:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766026.929977:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775766026.930701:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766026.930717:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775766026.930958:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766026.930960:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775766026.931152:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766026.931154:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775766026.931371:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766026.931373:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775766032.050003:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766032.050008:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775766032.050888:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766032.050894:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775766032.051463:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766032.051466:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775766032.051807:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766032.051808:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775766032.052093:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766032.052095:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775766037.170321:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766037.171052:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766037.171366:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766037.171640:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766037.171898:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766042.290749:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766042.290752:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775766042.291499:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766042.291502:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775766042.291830:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766042.291833:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775766042.292205:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766042.292207:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775766042.292452:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766042.292454:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775766047.409926:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766047.409932:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775766047.410471:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766047.410473:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775766047.410678:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766047.410680:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775766047.410894:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766047.410896:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775766047.411135:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766047.411137:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775766052.529784:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766052.529788:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775766052.530210:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766052.530212:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775766052.530419:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766052.530421:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775766052.530600:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766052.530602:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775766052.530775:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766052.530777:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1775766057.650986:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766057.650993:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775766057.652653:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766057.652662:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775766057.653505:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766057.653507:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775766057.653971:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766057.653973:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775766057.654348:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766057.654352:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775766062.770555:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766062.770567:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775766062.772013:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766062.772017:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775766062.773224:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766062.773235:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775766062.773949:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766062.773951:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775766062.774275:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766062.774277:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775766067.890129:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766067.890136:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775766067.891306:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766067.891315:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775766067.891807:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766067.891810:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775766067.892197:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766067.892200:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775766067.892514:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766067.892517:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775766073.010078:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766073.010084:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775766073.011066:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766073.011069:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775766073.011395:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766073.011397:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775766073.011749:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766073.011752:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775766073.012340:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766073.012342:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775766078.129794:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766078.129799:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775766078.130387:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766078.130388:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775766078.130576:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766078.130578:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775766078.130752:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766078.130754:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775766078.130929:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766078.130931:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775766083.249896:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766083.249903:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775766083.250568:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766083.250570:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775766083.250775:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766083.250777:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775766083.251046:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766083.251049:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775766083.251382:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766083.251384:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775766088.371060:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766088.371069:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775766088.372779:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766088.372783:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775766088.373259:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766088.373263:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775766088.373654:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766088.373657:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775766088.374230:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766088.374233:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775766093.489940:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766093.489946:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1775766093.491000:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766093.491007:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1775766093.491663:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766093.491668:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1775766093.491956:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766093.491958:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1775766093.492196:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766093.492198:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1775766098.610547:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766098.610557:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775766098.611875:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766098.611878:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775766098.612333:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766098.612336:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775766098.612683:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766098.612686:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775766098.613057:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766098.613060:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775766103.730105:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766103.730628:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766103.730996:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766103.731260:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766103.731460:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766108.850247:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766108.850256:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775766108.851137:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766108.851141:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775766108.851549:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766108.851552:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775766108.851938:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766108.851941:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775766108.852392:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766108.852394:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775766113.969852:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766113.969857:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775766113.970388:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766113.970391:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775766113.970788:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766113.970790:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775766113.971049:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766113.971051:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775766113.971299:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766113.971301:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775766119.091107:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766119.091126:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775766119.092771:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766119.092775:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775766119.093248:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766119.093251:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775766119.093742:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766119.093746:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775766119.094285:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766119.094288:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775766124.210080:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766124.210089:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775766124.211033:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766124.211036:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775766124.211702:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766124.211707:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775766124.211940:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766124.211942:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775766124.212141:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766124.212143:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775766129.329934:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766129.329940:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775766129.330545:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766129.330547:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775766129.330785:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766129.330786:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775766129.330969:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766129.330971:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775766129.331190:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766129.331192:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775766134.449845:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766134.449853:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775766134.450398:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766134.450400:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775766134.450636:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766134.450638:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775766134.450844:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766134.450847:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775766134.451043:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766134.451045:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775766139.569755:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766139.569759:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775766139.570374:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766139.570376:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775766139.570578:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766139.570580:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775766139.570751:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766139.570753:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775766139.571067:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766139.571069:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775766144.689955:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766144.689960:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775766144.690735:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766144.690740:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775766144.691017:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766144.691019:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775766144.691270:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766144.691272:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775766144.691618:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766144.691620:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775766149.810258:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766149.810268:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775766149.811517:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766149.811522:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775766149.811906:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766149.811910:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775766149.812368:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766149.812370:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775766149.812598:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766149.812600:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775766154.930312:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766154.930318:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775766154.931909:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766154.931916:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775766154.932650:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766154.932654:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775766154.933020:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766154.933021:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775766154.933227:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766154.933229:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775766160.050423:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766160.050429:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775766160.051733:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766160.051735:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775766160.052039:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766160.052042:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775766160.052394:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766160.052397:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775766160.052765:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766160.052768:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775766165.169933:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766165.169940:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775766165.170553:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766165.170555:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775766165.170746:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766165.170748:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775766165.170956:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766165.170959:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775766165.171242:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766165.171244:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775766170.290000:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766170.290536:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766170.290970:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766170.291187:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766170.291378:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766175.410571:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766175.410575:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775766175.410921:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766175.410923:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775766175.411232:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766175.411235:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775766175.411566:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766175.411569:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775766175.411895:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766175.411898:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775766180.530405:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766180.530409:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1775766180.531403:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766180.531405:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1775766180.531598:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766180.531600:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1775766180.531777:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766180.531779:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1775766180.531982:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766180.531984:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1775766185.650500:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766185.650510:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1775766185.652500:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766185.652511:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1775766185.654034:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766185.654040:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1775766185.654873:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766185.654877:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1775766185.655382:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766185.655385:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1775766190.769913:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766190.769943:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775766190.770729:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766190.770732:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775766190.770993:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766190.770995:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775766190.771257:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766190.771259:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775766190.771581:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766190.771584:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775766195.889994:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766195.890002:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775766195.891097:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766195.891100:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775766195.891438:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766195.891440:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775766195.891739:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766195.891741:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775766195.892199:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766195.892202:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775766201.010472:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766201.010487:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775766201.011910:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766201.011912:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775766201.012214:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766201.012217:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775766201.012442:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766201.012444:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775766201.012740:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766201.012743:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775766206.130011:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766206.130023:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775766206.131098:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766206.131100:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775766206.131367:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766206.131370:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775766206.131601:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766206.131603:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775766206.131933:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766206.131936:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775766211.250482:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766211.250488:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775766211.251085:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766211.251087:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775766211.251295:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766211.251297:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775766211.251479:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766211.251481:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775766211.251744:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766211.251746:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775766216.370687:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766216.370699:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775766216.372378:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766216.372406:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775766216.373595:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766216.373598:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775766216.373847:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766216.373849:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775766216.374238:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766216.374240:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775766221.490375:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766221.490385:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1775766221.491426:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766221.491429:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1775766221.491833:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766221.491836:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1775766221.492104:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766221.492107:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1775766221.492410:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766221.492412:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1775766226.609901:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766226.609907:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1775766226.610519:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766226.610521:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1775766226.610729:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766226.610731:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1775766226.610953:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766226.610955:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1775766226.611263:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766226.611265:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1775766231.729863:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766231.729868:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775766231.730479:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766231.730481:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775766231.730726:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766231.730728:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775766231.730979:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766231.730982:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775766231.731176:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766231.731177:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775766236.850094:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766236.850785:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766236.850989:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766236.851203:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766236.851362:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766241.970146:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766241.970184:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775766241.971041:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766241.971044:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775766241.971353:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766241.971355:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775766241.971714:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766241.971717:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775766241.972240:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766241.972243:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775766247.089989:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766247.089996:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775766247.090952:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766247.090956:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775766247.091587:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766247.091592:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775766247.091836:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766247.091838:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775766247.092092:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766247.092094:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775766252.209810:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766252.209816:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775766252.210696:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766252.210698:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775766252.210893:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766252.210894:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775766252.211167:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766252.211170:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775766252.211461:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766252.211463:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775766257.330145:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766257.330148:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775766257.330590:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766257.330593:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775766257.330871:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766257.330873:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775766257.331193:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766257.331196:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775766257.331503:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766257.331505:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775766262.450216:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766262.450224:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775766262.450964:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766262.450966:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775766262.451221:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766262.451223:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775766262.451499:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766262.451502:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775766262.451829:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766262.451831:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775766267.570037:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766267.570047:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775766267.571036:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766267.571038:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775766267.571352:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766267.571354:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775766267.571636:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766267.571639:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775766267.572036:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766267.572039:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775766272.690819:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766272.690825:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775766272.691488:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766272.691490:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775766272.691789:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766272.691791:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775766272.692077:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766272.692080:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775766272.692387:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766272.692390:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775766277.810375:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766277.810386:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775766277.812046:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766277.812052:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775766277.812717:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766277.812719:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775766277.812948:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766277.812950:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775766277.813252:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766277.813254:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775766282.930200:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766282.930208:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775766282.931410:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766282.931412:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775766282.931613:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766282.931615:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775766282.931855:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766282.931857:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775766282.932282:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766282.932284:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775766288.049770:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766288.049776:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775766288.050344:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766288.050345:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775766288.050576:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766288.050577:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775766288.050757:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766288.050759:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775766288.050980:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766288.050983:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775766293.170242:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766293.170247:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775766293.171470:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766293.171473:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775766293.171772:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766293.171774:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775766293.172000:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766293.172002:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775766293.172314:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766293.172316:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775766298.290324:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766298.290332:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775766298.291246:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766298.291249:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775766298.291533:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766298.291535:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775766298.291792:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766298.291795:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775766298.292236:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766298.292238:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775766303.409995:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766303.410461:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766303.410733:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766303.410909:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766303.411057:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766308.530394:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766308.530403:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 57 secs later) 00000800:00000200:0.1:1775766308.532040:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766308.532049:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 57 secs later) 00000800:00000200:0.1:1775766308.532672:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766308.532675:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 57 secs later) 00000800:00000200:0.1:1775766308.533265:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766308.533268:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 57 secs later) 00000800:00000200:0.1:1775766308.533685:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766308.533686:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 57 secs later) 00000800:00000200:0.1:1775766313.649800:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766313.649805:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1775766313.650401:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766313.650406:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1775766313.650738:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766313.650740:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1775766313.651051:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766313.651053:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1775766313.651257:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766313.651259:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1775766318.770120:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766318.770128:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1775766318.771035:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766318.771038:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1775766318.771383:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766318.771386:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1775766318.771687:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766318.771691:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1775766318.772260:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766318.772262:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1775766323.889841:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766323.889846:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1775766323.890433:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766323.890436:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1775766323.890700:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766323.890702:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1775766323.890899:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766323.890902:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1775766323.891208:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766323.891209:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1775766329.009864:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766329.009872:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1775766329.010511:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766329.010514:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1775766329.010873:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766329.010876:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1775766329.011128:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766329.011129:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1775766329.011381:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766329.011383:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775766334.130301:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766334.130312:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775766334.131201:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766334.131203:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775766334.131392:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766334.131394:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775766334.131630:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766334.131632:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775766334.131879:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766334.131881:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775766339.249887:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766339.249892:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775766339.250807:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766339.250813:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775766339.251423:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766339.251425:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775766339.251707:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766339.251710:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775766339.252034:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766339.252036:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775766344.369931:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766344.369936:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775766344.370556:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766344.370559:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775766344.370765:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766344.370766:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775766344.370985:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766344.370987:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775766344.371183:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766344.371185:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775766349.489936:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766349.489944:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 16 secs later) 00000800:00000200:3.1:1775766349.490420:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766349.490422:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 16 secs later) 00000800:00000200:3.1:1775766349.490616:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766349.490618:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 16 secs later) 00000800:00000200:3.1:1775766349.490815:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766349.490817:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 16 secs later) 00000800:00000200:3.1:1775766349.490998:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766349.491000:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 16 secs later) 00000800:00000200:3.1:1775766354.609802:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766354.609806:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1775766354.610516:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766354.610521:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1775766354.610798:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766354.610801:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1775766354.611131:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766354.611133:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1775766354.611680:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766354.611683:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1775766359.729941:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766359.729958:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1775766359.730601:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766359.730603:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1775766359.730811:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766359.730813:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1775766359.731036:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766359.731038:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1775766359.731259:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766359.731261:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1775766364.850614:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766364.850629:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1775766364.852325:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766364.852335:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1775766364.852771:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766364.852773:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1775766364.853020:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766364.853022:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1775766364.853249:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766364.853250:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1775766369.970658:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766369.971452:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766369.972262:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766369.973713:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766369.973875:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766375.090501:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766375.090511:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775766375.091978:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766375.091981:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775766375.092414:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766375.092417:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775766375.092727:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766375.092730:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775766375.092931:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766375.092933:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775766380.210151:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766380.210157:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775766380.211027:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766380.211030:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775766380.211392:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766380.211394:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775766380.211641:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766380.211643:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775766380.211977:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766380.211980:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775766385.330372:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766385.330381:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775766385.331648:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766385.331650:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775766385.331989:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766385.331992:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775766385.332374:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766385.332378:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775766385.332889:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766385.332892:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775766390.449891:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766390.449898:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775766390.450538:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766390.450540:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775766390.450791:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766390.450794:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775766390.450994:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766390.450996:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775766390.451246:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766390.451249:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775766395.570780:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766395.570791:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1775766395.572378:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766395.572400:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1775766395.572828:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766395.572831:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1775766395.573277:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766395.573281:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1775766395.573873:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766395.573876:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1775766400.689925:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766400.689934:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775766400.690829:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766400.690833:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775766400.691417:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766400.691421:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775766400.691691:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766400.691693:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775766400.691993:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766400.691995:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775766405.809995:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766405.810002:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775766405.810809:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766405.810811:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775766405.811140:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766405.811142:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775766405.811417:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766405.811419:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775766405.811702:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766405.811705:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775766410.930446:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766410.930458:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775766410.931236:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766410.931239:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775766410.931557:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766410.931560:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775766410.932054:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766410.932058:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775766410.932563:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766410.932567:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775766416.049922:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766416.049928:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775766416.050567:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766416.050568:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775766416.050793:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766416.050795:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775766416.051006:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766416.051008:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775766416.051228:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766416.051230:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775766421.169881:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766421.169887:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775766421.170580:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766421.170582:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775766421.170821:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766421.170823:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775766421.171047:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766421.171049:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775766421.171364:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766421.171366:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775766426.290327:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766426.290336:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775766426.291134:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766426.291136:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775766426.291444:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766426.291447:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775766426.291743:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766426.291745:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775766426.291996:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766426.291998:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775766431.410134:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766431.410142:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775766431.411279:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766431.411285:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775766431.411973:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766431.411975:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775766431.412212:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766431.412214:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775766431.412554:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766431.412557:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775766436.529906:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766436.530337:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766436.530530:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766436.530728:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766436.530888:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766441.650258:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766441.650265:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775766441.650985:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766441.650988:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775766441.651320:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766441.651323:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775766441.651653:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766441.651656:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775766441.651990:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766441.651993:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775766446.769915:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766446.769922:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775766446.770504:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766446.770505:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775766446.770701:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766446.770703:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775766446.770940:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766446.770941:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775766446.771138:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766446.771140:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775766451.890002:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766451.890007:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775766451.890679:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766451.890682:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775766451.891018:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766451.891020:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775766451.891286:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766451.891288:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775766451.891567:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766451.891569:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775766457.010846:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766457.010867:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775766457.012042:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766457.012046:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775766457.012428:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766457.012430:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775766457.012715:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766457.012717:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775766457.013049:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766457.013051:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775766462.130709:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766462.130720:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775766462.132389:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766462.132394:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775766462.132767:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766462.132769:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775766462.133067:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766462.133069:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775766462.133395:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766462.133397:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775766467.249825:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766467.249832:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775766467.250407:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766467.250409:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775766467.250606:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766467.250607:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775766467.250782:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766467.250784:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775766467.250958:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766467.250960:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1775766472.369799:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766472.369802:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1775766472.370439:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766472.370443:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1775766472.371052:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766472.371053:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1775766472.371329:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766472.371330:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1775766472.371551:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766472.371553:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1775766477.490191:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766477.490194:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775766477.491091:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766477.491093:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775766477.491355:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766477.491357:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775766477.491573:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766477.491575:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775766477.491836:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766477.491838:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775766482.610452:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766482.610470:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775766482.611648:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766482.611651:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775766482.612115:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766482.612118:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775766482.612595:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766482.612599:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775766482.612995:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766482.612999:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775766487.730568:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766487.730577:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775766487.731995:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766487.732020:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775766487.732305:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766487.732307:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775766487.732710:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766487.732715:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775766487.733148:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766487.733150:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775766492.850425:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766492.850435:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775766492.851823:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766492.851829:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775766492.852453:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766492.852458:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775766492.852920:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766492.852923:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775766492.853402:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766492.853405:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775766497.970186:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766497.970190:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775766497.971581:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766497.971584:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775766497.971788:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766497.971789:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775766497.971989:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766497.972008:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775766497.972290:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766497.972292:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775766503.089919:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766503.090153:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766503.090376:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766503.090553:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766503.090704:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766508.210118:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766508.210126:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775766508.210918:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766508.210922:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775766508.211318:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766508.211321:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775766508.211546:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766508.211548:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775766508.211799:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766508.211802:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775766513.329858:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766513.329864:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775766513.330468:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766513.330470:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775766513.330658:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766513.330659:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775766513.330898:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766513.330920:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775766513.331203:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766513.331205:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775766518.450110:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766518.450114:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775766518.450812:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766518.450815:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775766518.451134:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766518.451135:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775766518.451388:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766518.451389:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775766518.451592:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766518.451594:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775766523.570350:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766523.570359:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775766523.572105:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766523.572111:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775766523.572743:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766523.572749:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775766523.573128:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766523.573131:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775766523.573582:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766523.573586:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775766528.689773:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766528.689779:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775766528.690321:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766528.690323:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775766528.690528:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766528.690530:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775766528.690759:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766528.690761:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775766528.690939:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766528.690940:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775766533.809877:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766533.809884:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775766533.810608:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766533.810613:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775766533.810841:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766533.810842:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775766533.811085:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766533.811087:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775766533.811387:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766533.811389:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775766538.929893:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766538.929898:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775766538.930532:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766538.930533:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775766538.930762:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766538.930764:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775766538.931002:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766538.931004:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775766538.931339:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766538.931342:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775766544.050363:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766544.050366:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775766544.050708:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766544.050710:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775766544.050924:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766544.050926:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775766544.051242:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766544.051245:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775766544.051554:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766544.051557:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775766549.170403:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766549.170412:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775766549.171380:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766549.171383:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775766549.171692:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766549.171695:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775766549.172070:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766549.172073:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775766549.172521:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766549.172523:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775766554.290141:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766554.290148:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775766554.291230:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766554.291240:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775766554.291677:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766554.291679:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775766554.291859:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766554.291860:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775766554.292077:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766554.292079:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775766559.410571:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766559.410578:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775766559.412040:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766559.412045:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775766559.412375:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766559.412377:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775766559.412618:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766559.412620:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775766559.412934:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766559.412936:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775766564.529894:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766564.529897:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1775766564.530615:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766564.530616:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1775766564.530819:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766564.530822:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1775766564.531039:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766564.531041:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1775766564.531376:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766564.531378:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1775766569.650017:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766569.650458:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766569.650826:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766569.651062:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766569.651271:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766574.770007:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766574.770014:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775766574.770830:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766574.770833:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775766574.771225:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766574.771227:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775766574.771532:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766574.771534:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775766574.772041:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766574.772044:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775766579.890092:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766579.890114:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775766579.891226:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766579.891228:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775766579.891475:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766579.891477:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775766579.891704:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766579.891706:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775766579.892063:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766579.892066:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775766585.010223:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766585.010230:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775766585.011453:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766585.011458:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775766585.012361:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766585.012368:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775766585.012748:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766585.012752:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775766585.013228:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766585.013230:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775766590.130504:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766590.130514:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775766590.131885:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766590.131888:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775766590.132397:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766590.132401:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775766590.132766:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766590.132769:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775766590.133438:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766590.133442:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775766595.250007:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766595.250018:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775766595.251050:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766595.251059:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775766595.251427:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766595.251429:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775766595.251663:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766595.251665:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775766595.251838:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766595.251840:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775766600.370196:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766600.370206:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1775766600.371301:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766600.371304:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1775766600.371610:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766600.371612:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1775766600.372059:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766600.372062:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1775766600.372579:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766600.372582:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1775766605.490184:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766605.490231:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775766605.491097:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766605.491099:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775766605.491421:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766605.491423:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775766605.491627:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766605.491629:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775766605.491952:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766605.491955:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775766610.609891:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766610.609899:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775766610.610525:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766610.610527:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775766610.610762:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766610.610764:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775766610.610982:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766610.610984:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775766610.611317:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766610.611318:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775766615.730258:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766615.730266:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775766615.731414:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766615.731421:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775766615.731916:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766615.731918:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775766615.732215:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766615.732217:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775766615.732473:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766615.732475:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775766620.850188:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766620.850197:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775766620.850970:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766620.850972:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775766620.851224:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766620.851226:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775766620.851417:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766620.851419:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775766620.851595:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766620.851597:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775766625.970035:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766625.970044:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775766625.970898:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766625.970900:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775766625.971090:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766625.971091:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775766625.971316:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766625.971318:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775766625.971568:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766625.971570:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775766631.090136:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766631.090143:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775766631.091326:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766631.091330:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775766631.091671:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766631.091683:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775766631.091978:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766631.091980:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775766631.092268:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766631.092270:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775766636.209970:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766636.210336:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766636.210593:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766636.210783:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766636.210943:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766641.330298:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766641.330302:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775766641.330655:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766641.330658:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775766641.330960:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766641.330962:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775766641.331265:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766641.331267:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775766641.331688:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766641.331692:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775766646.450461:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766646.450473:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775766646.452312:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766646.452319:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775766646.453141:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766646.453146:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775766646.453960:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766646.453963:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775766646.454404:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766646.454405:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775766651.570245:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766651.570251:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775766651.570649:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766651.570651:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775766651.570842:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766651.570844:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775766651.571021:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766651.571035:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775766651.571245:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766651.571247:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1775766656.690072:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766656.690080:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1775766656.690732:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766656.690736:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1775766656.691049:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766656.691051:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1775766656.691426:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766656.691429:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1775766656.691699:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766656.691701:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1775766661.810111:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766661.810121:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1775766661.811202:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766661.811204:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1775766661.811395:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766661.811396:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1775766661.811579:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766661.811580:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1775766661.811825:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766661.811827:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1775766666.929873:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766666.929878:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775766666.930553:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766666.930556:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775766666.930774:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766666.930776:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775766666.931002:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766666.931004:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775766666.931366:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766666.931369:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775766672.050127:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766672.050131:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775766672.050830:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766672.050832:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775766672.051174:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766672.051176:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775766672.051371:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766672.051372:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775766672.051653:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766672.051655:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775766677.170453:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766677.170475:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775766677.172028:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766677.172031:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775766677.172758:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766677.172765:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775766677.173449:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766677.173451:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775766677.173644:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766677.173646:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775766682.290116:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766682.290137:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775766682.290818:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766682.290820:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775766682.291105:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766682.291107:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775766682.291355:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766682.291357:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775766682.291626:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766682.291628:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775766687.409858:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766687.409861:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775766687.410500:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766687.410502:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775766687.410789:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766687.410792:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775766687.411119:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766687.411121:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775766687.411464:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766687.411467:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775766692.530434:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766692.530444:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1775766692.531617:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766692.531620:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1775766692.532013:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766692.532018:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1775766692.532521:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766692.532525:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1775766692.532905:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766692.532908:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1775766697.650050:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766697.650057:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1775766697.650864:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766697.650866:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1775766697.651124:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766697.651126:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1775766697.651394:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766697.651396:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1775766697.651869:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766697.651872:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1775766702.770463:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766702.771227:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766702.771913:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766702.772331:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766702.772547:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766707.891317:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766707.891325:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775766707.891812:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766707.891814:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775766707.892312:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766707.892314:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775766707.892610:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766707.892612:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775766707.893024:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766707.893025:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775766713.010557:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766713.010569:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775766713.011889:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766713.011893:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775766713.012285:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766713.012288:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775766713.012637:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766713.012659:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775766713.013137:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766713.013141:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775766718.130146:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766718.130151:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775766718.131119:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766718.131122:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775766718.131462:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766718.131464:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775766718.131727:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766718.131729:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775766718.132077:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766718.132079:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775766723.249834:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766723.249841:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775766723.250461:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766723.250463:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775766723.250704:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766723.250706:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775766723.250883:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766723.250884:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775766723.251102:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766723.251104:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775766728.369862:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766728.369867:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775766728.370541:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766728.370542:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775766728.370783:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766728.370785:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775766728.371035:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766728.371037:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775766728.371288:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766728.371291:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775766733.489836:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766733.489840:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775766733.490540:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766733.490542:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775766733.490724:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766733.490726:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775766733.490898:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766733.490900:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775766733.491185:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766733.491187:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775766738.609973:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766738.609980:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775766738.611097:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766738.611104:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775766738.611721:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766738.611724:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775766738.612047:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766738.612049:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775766738.612460:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766738.612463:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775766743.729771:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766743.729775:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775766743.730444:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766743.730446:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775766743.730638:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766743.730639:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775766743.730815:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766743.730817:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775766743.731080:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766743.731082:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775766748.849784:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766748.849790:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775766748.850495:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766748.850497:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775766748.850683:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766748.850685:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775766748.850932:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766748.850935:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775766748.851224:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766748.851226:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775766753.969757:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766753.969763:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775766753.970392:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766753.970395:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775766753.970650:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766753.970652:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775766753.970854:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766753.970856:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775766753.971148:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766753.971150:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775766759.090234:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766759.090239:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775766759.090729:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766759.090731:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775766759.091141:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766759.091145:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775766759.091603:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766759.091606:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775766759.091974:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766759.091992:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775766764.210517:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766764.210528:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775766764.211617:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766764.211619:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775766764.211855:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766764.211857:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775766764.212055:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766764.212058:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775766764.212361:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766764.212363:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775766769.330043:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766769.330599:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766769.330998:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766769.331413:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766769.331735:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766774.450431:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766774.450437:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775766774.451503:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766774.451512:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775766774.451994:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766774.452012:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775766774.452611:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766774.452614:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775766774.452918:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766774.452921:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775766779.569774:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766779.569779:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1775766779.570231:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766779.570233:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1775766779.570440:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766779.570442:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1775766779.570643:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766779.570645:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1775766779.570894:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766779.570896:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1775766784.690120:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766784.690127:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1775766784.690824:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766784.690826:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1775766784.691196:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766784.691199:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1775766784.691434:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766784.691436:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1775766784.691721:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766784.691723:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1775766789.809672:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766789.809676:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1775766789.810108:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766789.810110:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1775766789.810346:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766789.810348:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1775766789.810526:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766789.810528:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1775766789.810769:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766789.810771:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775766794.930625:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766794.930633:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775766794.931425:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766794.931427:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775766794.931705:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766794.931708:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775766794.932000:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766794.932003:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775766794.932345:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766794.932347:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775766800.050386:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766800.050394:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775766800.051849:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766800.051858:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775766800.052844:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766800.052849:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775766800.053413:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766800.053416:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775766800.053946:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766800.053950:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775766805.169940:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766805.169946:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775766805.170690:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766805.170694:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775766805.170964:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766805.170966:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775766805.171299:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766805.171303:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775766805.171755:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766805.171757:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775766810.289863:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766810.289868:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775766810.290580:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766810.290581:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775766810.290805:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766810.290807:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775766810.290995:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766810.290997:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775766810.291203:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766810.291205:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775766815.409912:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766815.409917:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775766815.410804:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766815.410806:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775766815.411046:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766815.411047:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775766815.411276:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766815.411278:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775766815.411668:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766815.411670:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775766820.529854:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766820.529860:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 11 secs later) 00000800:00000200:0.1:1775766820.530551:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766820.530554:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 11 secs later) 00000800:00000200:0.1:1775766820.531066:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766820.531068:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 11 secs later) 00000800:00000200:0.1:1775766820.531307:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766820.531309:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 11 secs later) 00000800:00000200:0.1:1775766820.531564:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766820.531567:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1775766825.650661:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766825.650671:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1775766825.651967:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766825.651970:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1775766825.652278:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766825.652281:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1775766825.652668:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766825.652671:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1775766825.652984:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766825.652986:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1775766830.770218:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766830.770227:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775766830.771691:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766830.771697:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775766830.772749:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766830.772761:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775766830.773309:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766830.773311:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775766830.773630:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766830.773631:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775766835.889850:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766835.890411:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766835.890686:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766835.890866:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766835.891018:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766841.009837:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766841.009843:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775766841.010681:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766841.010687:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775766841.011061:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766841.011062:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775766841.011396:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766841.011399:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775766841.011728:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766841.011730:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775766846.129970:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766846.129976:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775766846.131014:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766846.131015:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775766846.131231:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766846.131233:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775766846.131499:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766846.131501:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775766846.131763:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766846.131764:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775766851.250266:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766851.250276:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775766851.250989:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766851.250992:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775766851.251461:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766851.251464:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775766851.251864:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766851.251868:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775766851.252274:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766851.252277:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775766856.369958:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766856.369962:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775766856.370668:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766856.370671:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775766856.370864:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766856.370866:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775766856.371034:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766856.371035:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775766856.371317:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766856.371320:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775766861.489821:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766861.489826:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775766861.490632:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766861.490637:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775766861.491142:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766861.491147:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775766861.491653:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766861.491655:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775766861.491830:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766861.491832:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775766866.610239:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766866.610244:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775766866.610593:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766866.610595:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775766866.610778:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766866.610780:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775766866.610992:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766866.610995:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775766866.611273:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766866.611275:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775766871.729774:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766871.729778:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775766871.730510:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766871.730513:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775766871.730829:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766871.730831:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775766871.731358:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766871.731361:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775766871.731681:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766871.731684:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775766876.849759:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766876.849765:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775766876.850425:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766876.850428:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775766876.850823:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766876.850825:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775766876.851085:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766876.851088:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775766876.851329:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766876.851331:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775766881.969786:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766881.969791:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775766881.970518:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766881.970520:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775766881.970724:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766881.970725:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775766881.970941:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766881.970944:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775766881.971243:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775766881.971245:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775766887.089896:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766887.089904:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775766887.090695:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766887.090698:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775766887.090952:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766887.090953:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775766887.091198:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766887.091200:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775766887.091489:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775766887.091490:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775766892.210702:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766892.210717:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775766892.212936:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766892.212943:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775766892.214082:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766892.214085:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775766892.214526:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766892.214529:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775766892.214928:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766892.214931:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775766897.330163:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766897.330193:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775766897.331204:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766897.331211:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775766897.331886:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766897.331906:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775766897.332197:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766897.332199:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775766897.332431:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766897.332433:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775766902.450074:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766902.450323:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766902.450449:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766902.450700:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766902.450924:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766907.570855:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766907.570869:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775766907.572364:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766907.572368:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775766907.572802:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766907.572805:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775766907.573056:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766907.573058:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775766907.573297:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766907.573300:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775766912.690193:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766912.690202:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775766912.691264:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766912.691266:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775766912.691512:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766912.691514:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775766912.691774:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766912.691776:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775766912.692096:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766912.692099:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775766917.810436:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766917.810442:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775766917.811376:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766917.811379:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775766917.811601:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766917.811603:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775766917.811784:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766917.811785:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775766917.812117:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766917.812119:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775766922.929901:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766922.929907:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775766922.930817:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766922.930820:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775766922.931388:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766922.931392:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775766922.931708:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766922.931709:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775766922.931950:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766922.931951:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775766928.049964:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766928.049970:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775766928.050784:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766928.050787:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775766928.051075:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766928.051078:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775766928.051367:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766928.051369:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775766928.051780:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766928.051783:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775766933.170090:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766933.170105:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1775766933.170914:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766933.170916:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1775766933.171108:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766933.171109:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1775766933.171380:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766933.171382:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1775766933.171570:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766933.171572:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1775766938.290336:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766938.290338:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1775766938.290715:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766938.290717:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1775766938.290962:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766938.290965:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1775766938.291259:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766938.291261:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1775766938.291489:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766938.291491:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1775766943.409790:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766943.409796:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 23 secs later) 00000800:00000200:3.1:1775766943.410396:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766943.410397:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 23 secs later) 00000800:00000200:3.1:1775766943.410578:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766943.410579:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 23 secs later) 00000800:00000200:3.1:1775766943.410744:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766943.410746:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 23 secs later) 00000800:00000200:3.1:1775766943.410976:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766943.410977:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 23 secs later) 00000800:00000200:1.1:1775766948.530049:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766948.530056:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775766948.530850:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766948.530851:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775766948.531066:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766948.531067:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775766948.531306:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766948.531308:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775766948.531690:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766948.531692:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775766953.650084:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766953.650092:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775766953.651223:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766953.651229:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775766953.651738:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766953.651742:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775766953.652019:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766953.652021:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775766953.652312:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766953.652315:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775766958.769760:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766958.769767:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775766958.770410:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766958.770411:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775766958.770627:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766958.770628:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775766958.770808:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766958.770810:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775766958.771152:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766958.771168:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775766963.889763:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766963.889770:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775766963.890467:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766963.890469:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775766963.890653:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766963.890654:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775766963.890870:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766963.890872:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775766963.891145:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766963.891147:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775766969.010324:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766969.010879:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766969.011698:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766969.012209:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766969.012351:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766974.130459:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766974.130464:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775766974.131715:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766974.131718:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775766974.132029:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766974.132031:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775766974.132393:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766974.132396:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775766974.132637:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766974.132639:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775766979.250193:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766979.250201:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775766979.251258:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766979.251263:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775766979.251618:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766979.251621:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775766979.251974:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766979.251977:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775766979.252182:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775766979.252195:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775766984.370089:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766984.370096:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775766984.371061:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766984.371066:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775766984.371715:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766984.371717:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775766984.371940:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766984.371942:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775766984.372208:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766984.372210:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775766989.490057:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766989.490064:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775766989.491222:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766989.491223:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775766989.491433:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766989.491436:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775766989.491636:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766989.491638:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775766989.491906:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766989.491908:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775766994.609889:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766994.609897:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775766994.610426:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766994.610428:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775766994.610664:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766994.610665:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775766994.610919:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766994.610921:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775766994.611121:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766994.611123:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775766999.730228:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766999.730237:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775766999.731567:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766999.731571:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775766999.731825:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766999.731826:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775766999.732074:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766999.732075:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775766999.732303:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775766999.732305:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775767004.849830:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767004.849835:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775767004.850399:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767004.850401:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775767004.850602:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767004.850604:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775767004.850789:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767004.850791:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775767004.851004:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767004.851007:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775767009.969854:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767009.969859:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775767009.970610:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767009.970614:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775767009.970956:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767009.970958:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775767009.971251:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767009.971253:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775767009.971427:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767009.971429:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775767015.089910:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767015.089917:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775767015.090776:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767015.090779:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775767015.091306:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767015.091309:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775767015.091660:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767015.091662:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775767015.091966:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767015.091968:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775767020.209928:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767020.209934:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775767020.210613:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767020.210628:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775767020.210882:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767020.210884:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775767020.211126:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767020.211128:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775767020.211450:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767020.211452:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775767025.329806:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767025.329811:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775767025.330485:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767025.330487:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775767025.330679:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767025.330681:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775767025.330873:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767025.330875:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775767025.331151:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767025.331153:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775767030.449832:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767030.449836:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775767030.450493:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767030.450495:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775767030.450679:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767030.450680:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775767030.450894:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767030.450907:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775767030.451212:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767030.451213:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775767035.569735:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767035.570105:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767035.570437:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767035.570588:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767035.570737:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767040.690090:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767040.690098:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775767040.691021:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767040.691024:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775767040.691407:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767040.691410:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775767040.691802:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767040.691805:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775767040.692128:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767040.692130:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775767045.809906:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767045.809913:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775767045.810837:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767045.810840:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775767045.811336:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767045.811339:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775767045.811698:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767045.811699:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775767045.811948:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767045.811950:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775767050.930045:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767050.930053:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775767050.930617:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767050.930619:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775767050.930864:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767050.930866:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775767050.931060:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767050.931062:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775767050.931316:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767050.931318:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775767056.049780:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767056.049783:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775767056.050373:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767056.050375:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775767056.050605:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767056.050606:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775767056.050781:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767056.050782:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775767056.050948:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767056.050949:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775767061.170128:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767061.170134:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775767061.170986:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767061.170988:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775767061.171243:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767061.171245:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775767061.171467:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767061.171468:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775767061.171643:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767061.171645:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775767066.290014:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767066.290022:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775767066.290538:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767066.290541:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775767066.290836:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767066.290839:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775767066.291180:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767066.291183:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775767066.291512:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767066.291516:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1775767071.410015:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767071.410022:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1775767071.410755:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767071.410757:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1775767071.411032:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767071.411034:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1775767071.411284:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767071.411285:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1775767071.411620:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767071.411623:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1775767076.530009:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767076.530014:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775767076.530932:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767076.530935:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775767076.531492:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767076.531493:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775767076.531676:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767076.531695:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775767076.531968:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767076.531970:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775767081.650387:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767081.650396:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775767081.651200:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767081.651203:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775767081.651506:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767081.651509:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775767081.651810:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767081.651812:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775767081.652117:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767081.652122:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775767086.769813:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767086.769818:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775767086.770511:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767086.770513:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775767086.770786:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767086.770788:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775767086.771021:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767086.771022:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775767086.771314:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767086.771318:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775767091.890095:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767091.890105:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775767091.891084:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767091.891087:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775767091.891331:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767091.891333:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775767091.891550:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767091.891552:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775767091.891907:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767091.891909:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775767097.009752:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767097.009757:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775767097.010399:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767097.010404:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775767097.010610:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767097.010611:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775767097.010979:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767097.010981:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775767097.011277:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767097.011278:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775767102.130215:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767102.131091:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767102.131610:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767102.132118:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767102.132328:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767107.250289:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767107.250296:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775767107.251683:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767107.251687:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775767107.252540:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767107.252549:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775767107.253590:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767107.253596:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775767107.254626:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767107.254629:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775767112.369771:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767112.369791:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775767112.370424:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767112.370426:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775767112.370645:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767112.370647:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775767112.370881:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767112.370883:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775767112.371176:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767112.371177:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775767117.489924:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767117.489931:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1775767117.490548:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767117.490550:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1775767117.490772:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767117.490774:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1775767117.490955:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767117.490957:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1775767117.491226:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767117.491228:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1775767122.610351:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767122.610378:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775767122.611149:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767122.611153:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775767122.611634:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767122.611638:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775767122.612017:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767122.612021:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775767122.612465:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767122.612468:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775767127.729897:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767127.729902:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775767127.730484:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767127.730486:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775767127.730755:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767127.730756:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775767127.731004:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767127.731006:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775767127.731256:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767127.731258:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775767132.850348:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767132.850358:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775767132.851469:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767132.851473:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775767132.851727:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767132.851729:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775767132.851945:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767132.851947:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775767132.852219:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767132.852221:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775767137.970399:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767137.970410:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775767137.972017:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767137.972031:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775767137.972695:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767137.972699:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775767137.973032:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767137.973034:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775767137.973310:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767137.973312:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775767143.089800:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767143.089804:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775767143.090468:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767143.090474:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775767143.090926:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767143.090929:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775767143.091341:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767143.091344:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775767143.091619:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767143.091622:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775767148.209786:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767148.209792:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775767148.210514:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767148.210516:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775767148.210697:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767148.210699:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775767148.210874:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767148.210875:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775767148.211193:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767148.211195:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775767153.329713:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767153.329717:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775767153.330336:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767153.330340:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775767153.330662:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767153.330664:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775767153.330933:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767153.330935:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775767153.331182:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767153.331185:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775767158.450081:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767158.450089:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775767158.451247:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767158.451256:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775767158.451785:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767158.451787:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775767158.452151:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767158.452152:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775767158.452404:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767158.452406:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775767163.570263:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767163.570274:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775767163.571542:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767163.571543:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775767163.571733:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767163.571735:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775767163.571902:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767163.571904:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775767163.572197:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767163.572199:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1775767168.689994:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767168.690467:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767168.690928:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767168.691303:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767168.691644:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767173.809941:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767173.809948:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775767173.810747:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767173.810750:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775767173.811052:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767173.811054:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775767173.811325:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767173.811327:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775767173.811556:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767173.811559:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775767178.930223:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767178.930228:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775767178.931175:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767178.931177:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775767178.931399:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767178.931401:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775767178.931583:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767178.931584:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775767178.931754:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767178.931755:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775767184.049789:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767184.049792:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775767184.050485:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767184.050487:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775767184.050777:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767184.050778:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775767184.051014:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767184.051016:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775767184.051207:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767184.051209:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775767189.170123:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767189.170130:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775767189.170757:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767189.170760:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775767189.171104:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767189.171107:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775767189.171449:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767189.171453:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775767189.171774:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767189.171776:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775767194.290416:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767194.290428:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775767194.291454:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767194.291457:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775767194.291796:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767194.291798:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775767194.292113:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767194.292116:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775767194.292592:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767194.292595:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775767199.410065:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767199.410073:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775767199.411007:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767199.411015:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775767199.411672:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767199.411674:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775767199.411921:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767199.411923:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775767199.412140:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767199.412142:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775767204.529817:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767204.529824:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775767204.530500:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767204.530502:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775767204.530765:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767204.530767:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775767204.531039:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767204.531042:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775767204.531311:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767204.531313:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775767209.650049:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767209.650058:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775767209.651114:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767209.651116:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775767209.651379:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767209.651381:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775767209.651635:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767209.651638:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775767209.651912:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767209.651914:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775767214.770073:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767214.770079:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775767214.771105:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767214.771108:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775767214.771399:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767214.771401:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775767214.771629:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767214.771631:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775767214.771939:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767214.771941:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775767219.890042:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767219.890053:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775767219.890923:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767219.890925:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775767219.891119:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767219.891121:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775767219.891408:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767219.891411:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775767219.891737:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767219.891740:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775767225.010074:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767225.010079:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775767225.010724:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767225.010726:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775767225.010946:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767225.010948:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775767225.011122:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767225.011124:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775767225.011348:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767225.011350:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775767230.130014:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767230.130022:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775767230.130993:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767230.130999:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775767230.131602:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767230.131605:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775767230.131903:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767230.131905:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775767230.132224:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767230.132227:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775767235.250138:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767235.250770:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767235.250952:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767235.251284:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767235.251471:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767240.370668:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767240.370671:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775767240.371064:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767240.371067:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775767240.371435:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767240.371437:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775767240.371900:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767240.371903:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775767240.372286:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767240.372288:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775767245.490084:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767245.490097:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1775767245.491113:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767245.491115:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1775767245.491426:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767245.491428:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1775767245.491623:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767245.491625:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1775767245.491875:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767245.491877:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 52 secs later) 00000800:00000200:3.1:1775767250.609938:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767250.609945:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775767250.610500:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767250.610502:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775767250.610826:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767250.610828:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775767250.611183:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767250.611185:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775767250.611495:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767250.611498:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775767255.730268:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767255.730279:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775767255.731362:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767255.731365:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775767255.731701:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767255.731703:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775767255.731938:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767255.731940:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775767255.732207:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767255.732209:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1775767260.851010:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767260.851016:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1775767260.851807:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767260.851808:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1775767260.852008:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767260.852009:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1775767260.852215:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767260.852217:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1775767260.852497:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767260.852499:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775767265.969740:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767265.969744:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775767265.970405:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767265.970407:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775767265.970620:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767265.970621:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775767265.970796:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767265.970797:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775767265.971060:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767265.971062:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775767271.089753:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767271.089757:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775767271.090573:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767271.090576:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775767271.090775:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767271.090776:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775767271.090999:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767271.091001:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775767271.091212:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767271.091214:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775767276.209837:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767276.209870:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775767276.210495:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767276.210497:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775767276.210711:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767276.210713:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775767276.210984:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767276.210987:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775767276.211294:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767276.211296:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775767281.329988:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767281.329995:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775767281.330683:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767281.330686:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775767281.330962:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767281.330965:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775767281.331244:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767281.331246:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775767281.331504:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767281.331506:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775767286.450043:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767286.450050:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775767286.450863:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767286.450866:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775767286.451203:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767286.451205:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775767286.451508:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767286.451510:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775767286.451938:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767286.451941:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775767291.569996:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767291.570002:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1775767291.570804:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767291.570807:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1775767291.572381:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767291.572384:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1775767291.572635:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767291.572636:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1775767291.572919:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767291.572921:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1775767296.690221:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767296.690227:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1775767296.690633:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767296.690635:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1775767296.690877:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767296.690879:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1775767296.691097:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767296.691098:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1775767296.691316:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767296.691318:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1775767301.809768:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767301.809999:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767301.810170:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767301.810344:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767301.810519:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767306.929938:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767306.929941:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775767306.930726:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767306.930731:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775767306.931102:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767306.931118:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775767306.931567:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767306.931571:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775767306.931863:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767306.931866:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775767312.049783:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767312.049787:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775767312.050351:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767312.050352:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775767312.050581:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767312.050583:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775767312.050758:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767312.050760:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775767312.050989:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767312.050991:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775767317.169980:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767317.169988:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775767317.170733:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767317.170736:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775767317.170944:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767317.170946:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775767317.171117:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767317.171119:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775767317.171399:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767317.171414:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775767322.290011:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767322.290020:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775767322.290935:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767322.290939:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775767322.291529:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767322.291531:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775767322.291715:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767322.291717:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775767322.292064:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767322.292066:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775767327.409890:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767327.409894:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775767327.410419:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767327.410420:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775767327.410612:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767327.410614:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775767327.410785:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767327.410787:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775767327.411038:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767327.411041:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775767332.529733:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767332.529735:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775767332.530297:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767332.530300:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775767332.530836:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767332.530839:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775767332.531143:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767332.531146:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775767332.531429:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767332.531430:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775767337.649799:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767337.649803:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775767337.650499:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767337.650503:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775767337.650714:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767337.650715:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775767337.650931:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767337.650933:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775767337.651262:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767337.651264:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775767342.769883:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767342.769889:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775767342.770499:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767342.770501:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775767342.770686:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767342.770688:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775767342.770858:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767342.770859:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775767342.771025:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767342.771027:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775767347.890438:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767347.890443:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775767347.891511:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767347.891513:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775767347.891746:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767347.891748:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775767347.891938:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767347.891940:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775767347.892308:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767347.892310:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775767353.009896:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767353.009902:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775767353.010966:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767353.010971:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775767353.011498:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767353.011500:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775767353.011797:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767353.011799:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775767353.012248:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767353.012251:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775767358.129791:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767358.129797:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775767358.130426:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767358.130428:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775767358.130608:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767358.130609:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775767358.130776:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767358.130778:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775767358.131034:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767358.131036:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775767363.249702:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767363.249706:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775767363.250482:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767363.250485:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775767363.250783:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767363.250786:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775767363.250996:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767363.250998:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775767363.251316:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767363.251319:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775767368.369672:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767368.370343:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767368.370552:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767368.370687:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767368.371406:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767373.489874:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767373.489881:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 57 secs later) 00000800:00000200:0.1:1775767373.490686:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767373.490688:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 57 secs later) 00000800:00000200:0.1:1775767373.491033:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767373.491036:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 57 secs later) 00000800:00000200:0.1:1775767373.491373:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767373.491376:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 57 secs later) 00000800:00000200:0.1:1775767373.491822:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767373.491825:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 57 secs later) 00000800:00000200:0.1:1775767378.609979:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767378.609983:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1775767378.610675:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767378.610677:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1775767378.610867:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767378.610868:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1775767378.611153:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767378.611176:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1775767378.611576:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767378.611578:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1775767383.730236:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767383.730243:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1775767383.731595:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767383.731602:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1775767383.732567:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767383.732571:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1775767383.733386:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767383.733389:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1775767383.733685:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767383.733688:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1775767388.849911:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767388.849918:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1775767388.850578:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767388.850580:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1775767388.850773:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767388.850775:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1775767388.851062:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767388.851064:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1775767388.851378:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767388.851380:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1775767393.969878:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767393.969883:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1775767393.970489:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767393.970491:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1775767393.970696:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767393.970697:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1775767393.970926:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767393.970929:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1775767393.971200:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767393.971202:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1775767399.089888:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767399.089893:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775767399.090538:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767399.090541:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775767399.090845:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767399.090847:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775767399.091133:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767399.091136:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775767399.091437:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767399.091439:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775767404.210230:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767404.210240:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775767404.211385:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767404.211387:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775767404.211629:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767404.211631:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775767404.211838:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767404.211841:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775767404.212110:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767404.212112:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775767409.329994:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767409.330001:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775767409.330674:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767409.330676:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775767409.330872:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767409.330874:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775767409.331116:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767409.331118:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775767409.331391:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767409.331394:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775767414.449976:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767414.449982:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775767414.450920:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767414.450925:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775767414.451469:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767414.451472:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775767414.451786:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767414.451788:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775767414.452044:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767414.452046:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775767419.570196:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767419.570202:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 11 secs later) 00000800:00000200:0.1:1775767419.571301:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767419.571303:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 11 secs later) 00000800:00000200:0.1:1775767419.571547:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767419.571549:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 11 secs later) 00000800:00000200:0.1:1775767419.571742:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767419.571744:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 11 secs later) 00000800:00000200:0.1:1775767419.572036:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767419.572038:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 11 secs later) 00000800:00000200:0.1:1775767424.689968:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767424.689974:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1775767424.690737:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767424.690740:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1775767424.691070:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767424.691073:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1775767424.691539:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767424.691542:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1775767424.691848:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767424.691851:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1775767429.810008:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767429.810012:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1775767429.810631:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767429.810633:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1775767429.810856:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767429.810859:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1775767429.811080:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767429.811082:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1775767429.811343:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767429.811345:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1775767434.930093:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767434.930702:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767434.930968:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767434.931196:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767434.931372:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767440.050110:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767440.050118:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775767440.050962:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767440.050964:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775767440.051288:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767440.051291:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775767440.051750:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767440.051753:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775767440.052851:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767440.052854:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775767445.169977:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767445.169982:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775767445.170970:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767445.170974:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775767445.171401:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767445.171403:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775767445.171581:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767445.171583:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775767445.171752:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767445.171753:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775767450.290228:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767450.290232:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775767450.290541:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767450.290543:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775767450.290756:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767450.290758:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775767450.290963:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767450.290965:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775767450.291262:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767450.291265:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775767455.409757:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767455.409761:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775767455.410389:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767455.410390:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775767455.410615:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767455.410617:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775767455.410790:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767455.410792:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775767455.410961:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767455.410962:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775767460.529782:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767460.529787:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1775767460.530409:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767460.530410:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1775767460.530650:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767460.530652:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1775767460.530825:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767460.530827:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1775767460.531081:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767460.531083:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1775767465.649804:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767465.649807:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775767465.650442:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767465.650444:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775767465.650666:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767465.650667:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775767465.650842:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767465.650844:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775767465.651107:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767465.651109:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775767470.770054:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767470.770060:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775767470.770778:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767470.770782:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775767470.771217:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767470.771219:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775767470.771532:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767470.771534:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775767470.771787:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767470.771789:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775767475.890647:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767475.890656:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775767475.892293:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767475.892302:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775767475.893067:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767475.893071:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775767475.893324:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767475.893326:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775767475.893702:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767475.893704:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775767481.009865:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767481.009870:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775767481.010582:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767481.010587:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775767481.010794:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767481.010796:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775767481.010992:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767481.010993:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775767481.011256:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767481.011258:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775767486.129914:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767486.129918:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775767486.130613:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767486.130615:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775767486.130859:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767486.130861:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775767486.131109:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767486.131112:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775767486.131458:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767486.131461:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775767491.250028:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767491.250036:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775767491.250677:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767491.250680:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775767491.251012:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767491.251015:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775767491.251362:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767491.251364:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775767491.251630:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767491.251632:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775767496.369809:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767496.369814:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775767496.370424:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767496.370426:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775767496.370628:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767496.370630:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775767496.370827:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767496.370829:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775767496.371073:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767496.371074:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775767501.490420:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767501.490856:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767501.491188:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767501.491432:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767501.491598:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767506.610934:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767506.610946:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775767506.612329:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767506.612338:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775767506.612986:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767506.612988:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775767506.613213:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767506.613215:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775767506.613400:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767506.613402:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775767511.729857:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767511.729861:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775767511.730524:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767511.730525:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775767511.730776:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767511.730779:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775767511.731027:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767511.731029:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775767511.731268:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767511.731270:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775767516.849816:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767516.849823:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775767516.850580:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767516.850582:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775767516.850952:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767516.850954:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775767516.851293:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767516.851296:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775767516.851878:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767516.851881:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775767521.969826:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767521.969831:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775767521.970467:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767521.970469:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775767521.970696:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767521.970698:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775767521.970921:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767521.970923:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775767521.971229:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767521.971232:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775767527.090069:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767527.090077:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775767527.090968:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767527.090971:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775767527.091311:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767527.091314:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775767527.091639:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767527.091641:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775767527.091938:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767527.091941:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775767532.210471:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767532.210479:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775767532.211472:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767532.211474:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775767532.211677:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767532.211678:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775767532.211857:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767532.211859:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775767532.212068:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767532.212071:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775767537.330676:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767537.330687:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1775767537.332424:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767537.332430:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1775767537.332927:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767537.332930:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1775767537.333342:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767537.333344:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1775767537.333627:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767537.333629:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1775767542.450197:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767542.450205:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 23 secs later) 00000800:00000200:0.1:1775767542.451082:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767542.451084:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 23 secs later) 00000800:00000200:0.1:1775767542.451344:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767542.451345:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 23 secs later) 00000800:00000200:0.1:1775767542.451529:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767542.451531:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 23 secs later) 00000800:00000200:0.1:1775767542.451709:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767542.451710:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 23 secs later) 00000800:00000200:0.1:1775767547.569857:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767547.569864:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775767547.570621:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767547.570623:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775767547.570848:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767547.570849:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775767547.571057:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767547.571059:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775767547.571418:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767547.571420:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775767552.689957:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767552.689961:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775767552.690561:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767552.690563:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775767552.690788:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767552.690790:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775767552.691003:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767552.691005:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775767552.691309:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767552.691311:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775767557.809818:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767557.809822:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775767557.810362:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767557.810364:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775767557.810566:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767557.810568:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775767557.810784:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767557.810786:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775767557.811072:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767557.811074:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775767562.930001:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767562.930009:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775767562.930822:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767562.930824:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775767562.931101:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767562.931104:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775767562.931422:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767562.931424:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775767562.931801:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767562.931803:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775767568.050257:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767568.050946:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767568.051630:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767568.052205:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767568.052702:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767573.170088:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767573.170096:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775767573.170869:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767573.170872:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775767573.171222:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767573.171225:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775767573.171524:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767573.171527:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775767573.171966:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767573.171968:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775767578.289989:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767578.289996:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775767578.290591:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767578.290593:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775767578.290901:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767578.290903:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775767578.291248:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767578.291251:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775767578.291555:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767578.291557:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775767583.410032:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767583.410037:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775767583.410424:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767583.410425:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775767583.410639:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767583.410641:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775767583.410859:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767583.410861:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775767583.411078:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767583.411081:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 02000000:00000001:3.0:1775767583.411310:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767583.411311:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767583.411312:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207889f80. 00000100:00000001:3.0:1775767583.411314:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767583.411315:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767583.411316:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767583.411318:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207888700 x1862022294209792/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767608 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767583.411321:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767583.411322:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767583.411323:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b2800. 02000000:00000001:3.0:1775767583.411324:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767583.411324:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767583.411325:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:3.0:1775767583.411326:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767583.411327:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543f800. 02000000:00000001:3.0:1775767583.411328:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767583.411329:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767583.411329:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207888700. 00000100:00000001:3.0:1775767583.411330:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767583.411331:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767583.411332:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767583.411335:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767583.411335:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767583.411337:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767583.411338:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767584.433283:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767584.433291:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767584.433294:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767584.433302:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767584.433307:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767584.433312:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767584.433316:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767584.433320:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767584.433323:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767584.497358:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775767584.497370:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775767584.497375:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775767585.457265:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767585.457271:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767585.457274:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767585.457281:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767585.457288:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767585.457292:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767585.457297:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767585.457301:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767585.457305:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767586.481276:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767586.481285:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767586.481288:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767586.481296:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767586.481303:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767586.481308:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767586.481321:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767586.481324:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767586.481328:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767586.545312:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775767586.545316:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775767586.545320:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775767587.505253:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767587.505259:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767587.505262:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767587.505269:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767587.505276:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767587.505280:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767587.505286:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767587.505289:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767587.505291:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767588.529266:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767588.529274:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767588.529279:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767588.529287:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767588.529294:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767588.529299:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767588.529314:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767588.529318:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767588.529335:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767588.529525:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767588.529529:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767588.529532:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767588.529542:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fce00 x1862022294209920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767588.529558:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767588.529561:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767588.529566:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767588.529570:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767588.529574:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767588.529579:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294209920:192.168.202.156@tcp:250:kworker.0 00000100:00000001:3.0:1775767588.529584:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767588.529588:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767588.529591:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767588.529596:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767588.529602:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b2800. 02000000:00000001:3.0:1775767588.529605:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767588.529613:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767588.529618:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b9a0. 00000400:00000010:3.0:1775767588.529626:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e088. 00000100:00000200:3.0:1775767588.529635:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294209920, portal 25 00000100:00000001:3.0:1775767588.529640:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767588.529642:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641200128 : -112966068351488 : ffff9942099fce00) 00000100:00000040:3.0:1775767588.529652:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fce00 x1862022294209920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767613 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767588.529665:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767588.529672:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767588.529676:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e660. 00000100:00000200:3.0:1775767588.529680:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294209920, offset 0 00000400:00000200:3.0:1775767588.529689:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767588.529702:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767588.529711:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767588.529720:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767588.529726:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767588.529733:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767588.529737:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767588.529743:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224813:224813:256:4294967295] 192.168.202.156@tcp LPNI seq info [224809:224809:8:4294967295] 00000400:00000200:3.0:1775767588.529749:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767588.529762:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767588.529772:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767588.529777:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8200. 00000800:00000200:3.0:1775767588.529787:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767588.529793:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775767588.529800:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767588.529805:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000100:3.0:1775767588.529810:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767588.529813:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8200. 00000400:00000200:3.0:1775767588.529819:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767588.529825:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767588.529832:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e660 00000400:00000010:3.0:1775767588.529835:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e660. 00000100:00000001:3.0:1775767588.529841:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767588.529846:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fce00 x1862022294209920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767613 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767588.529859:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767588.529865:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fce00 x1862022294209920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767613 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767588.529890:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767588.529893:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767588.529902:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767588.529910:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767588.529916:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299514685 00000100:00000010:3.0:1775767588.529924:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207981de0. 00000100:00000001:3.0:1775767588.529936:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1775767588.529944:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299514685 00000100:00000010:3.0:1775767588.529949:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942079817e0. 00000100:00000001:3.0:1775767588.529953:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1775767588.529957:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767588.529961:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767588.529966:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767588.529974:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767588/real 1775767588] req@ffff9942099fce00 x1862022294209920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767588.529990:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767588.529992:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767588.529995:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b9a0. 00000400:00000200:3.0:1775767588.529999:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e088 00000400:00000010:3.0:1775767588.530003:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e088. 00000100:00000001:3.0:1775767588.530007:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767588.530012:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fce00 x1862022294209920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767588.530029:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fce00 x1862022294209920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767588.530042:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767588.530044:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767588.530046:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767588.530049:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767588.530054:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fce00 x1862022294209920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767588.530067:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767588.530073:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fce00 x1862022294209920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767613 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767588.530084:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767588.530087:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767588.530090:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767588.530092:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767588.530097:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767588.530101:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767588.530108:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767588.530117:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767588.530122:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767588.530131:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fce00 x1862022294209920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767613 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767588.530152:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294209920:192.168.202.156@tcp:250:kworker.0 00000100:00000001:3.0:1775767588.530175:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767588.530177:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767588.530180:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fce00 x1862022294209920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767613 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767588.530186:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767588.530190:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767588.530194:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b2800. 02000000:00000001:3.0:1775767588.530199:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767588.530203:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767588.530207:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:3.0:1775767588.530212:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767588.530217:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029400. 02000000:00000001:3.0:1775767588.530223:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767588.530227:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767588.530231:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fce00. 00000100:00000001:3.0:1775767588.530237:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767588.530240:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767588.530245:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767588.530256:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767588.530258:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767588.530261:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767588.530264:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767588.530266:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767588.530272:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fdc00 x1862022294210048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767588.530284:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767588.530286:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767588.530289:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767588.530291:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767588.530294:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767588.530299:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294210048:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767588.530304:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767588.530307:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767588.530309:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767588.530310:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767588.530314:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b2800. 02000000:00000001:3.0:1775767588.530317:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767588.530324:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767588.530330:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b9a0. 00000400:00000010:3.0:1775767588.530338:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e088. 00000100:00000200:3.0:1775767588.530344:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294210048, portal 10 00000100:00000001:3.0:1775767588.530347:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767588.530351:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641203712 : -112966068347904 : ffff9942099fdc00) 00000100:00000040:3.0:1775767588.530362:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fdc00 x1862022294210048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767613 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767588.530376:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767588.530382:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767588.530387:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e660. 00000100:00000200:3.0:1775767588.530391:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294210048, offset 0 00000400:00000200:3.0:1775767588.530399:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767588.530407:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767588.530413:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767588.530418:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767588.530421:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767588.530424:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767588.530427:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767588.530432:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224814:224814:256:4294967295] 192.168.202.156@tcp LPNI seq info [224810:224810:8:4294967295] 00000400:00000200:3.0:1775767588.530435:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767588.530442:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767588.530445:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767588.530448:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8200. 00000800:00000200:3.0:1775767588.530451:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767588.530453:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775767588.530456:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767588.530458:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000100:3.0:1775767588.530460:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767588.530462:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8200. 00000400:00000200:3.0:1775767588.530464:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767588.530466:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767588.530469:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e660 00000400:00000010:3.0:1775767588.530470:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e660. 00000100:00000001:3.0:1775767588.530472:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767588.530475:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fdc00 x1862022294210048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767613 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767588.530481:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767588.530483:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fdc00 x1862022294210048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767613 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767588.530487:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767588.530488:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767588.530490:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767588.530494:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:3.0:1775767588.530496:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767588.530497:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299513363 00000100:00000001:3.0:1775767588.530499:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299514685 00000100:00000001:3.0:1775767588.530501:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299513363 00000100:00000001:3.0:1775767588.530502:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299514685 00000100:00000001:3.0:1775767588.530504:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767588.530506:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767588.530507:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767588.530510:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767588/real 1775767588] req@ffff9942099fdc00 x1862022294210048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767588.530515:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767588.530516:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767588.530517:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b9a0. 00000400:00000200:3.0:1775767588.530518:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e088 00000400:00000010:3.0:1775767588.530519:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e088. 00000100:00000001:3.0:1775767588.530521:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767588.530523:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fdc00 x1862022294210048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767588.530529:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fdc00 x1862022294210048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767588.530532:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767588.530533:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767588.530534:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767588.530535:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767588.530537:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fdc00 x1862022294210048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767588.530541:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767588.530544:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fdc00 x1862022294210048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767613 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767588.530548:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767588.530549:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767588.530551:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767588.530552:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767588.530553:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767588.530555:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767588.530557:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767588.530560:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767588.530561:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767588.530564:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fdc00 x1862022294210048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767613 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767588.530569:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294210048:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767588.530571:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767588.530573:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fe680 x1862022294210176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767588.530577:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767588.530578:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767588.530579:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767588.530580:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767588.530582:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767588.530583:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294210176:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767588.530586:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767588.530587:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767588.530588:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767588.530589:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767588.530592:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b2c00. 02000000:00000001:3.0:1775767588.530593:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767588.530595:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767588.530597:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b9a0. 00000400:00000010:3.0:1775767588.530598:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e088. 00000100:00000200:3.0:1775767588.530600:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294210176, portal 10 00000100:00000001:3.0:1775767588.530602:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767588.530603:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641206400 : -112966068345216 : ffff9942099fe680) 00000100:00000040:3.0:1775767588.530607:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fe680 x1862022294210176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767613 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767588.530612:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767588.530613:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767588.530615:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e660. 00000100:00000200:3.0:1775767588.530617:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294210176, offset 0 00000400:00000200:3.0:1775767588.530619:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767588.530622:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767588.530624:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767588.530626:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767588.530627:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767588.530629:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767588.530631:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767588.530633:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224815:224815:256:4294967295] 192.168.202.156@tcp LPNI seq info [224811:224811:8:4294967295] 00000400:00000200:3.0:1775767588.530636:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767588.530640:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767588.530643:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767588.530645:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8200. 00000800:00000200:3.0:1775767588.530647:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767588.530649:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775767588.530652:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767588.530653:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000100:3.0:1775767588.530656:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767588.530657:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8200. 00000400:00000200:3.0:1775767588.530659:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767588.530661:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767588.530663:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e660 00000400:00000010:3.0:1775767588.530664:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e660. 00000100:00000001:3.0:1775767588.530666:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767588.530668:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fe680 x1862022294210176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767613 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767588.530673:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767588.530675:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fe680 x1862022294210176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767613 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767588.530678:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767588.530680:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767588.530682:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767588.530683:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:3.0:1775767588.530686:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767588.530687:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299513355 00000100:00000001:3.0:1775767588.530689:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299514685 00000100:00000001:3.0:1775767588.530691:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299513355 00000100:00000001:3.0:1775767588.530692:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299514685 00000100:00000001:3.0:1775767588.530693:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767588.530694:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767588.530695:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767588.530697:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767588/real 1775767588] req@ffff9942099fe680 x1862022294210176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767588.530701:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767588.530702:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767588.530703:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b9a0. 00000400:00000200:3.0:1775767588.530704:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e088 00000400:00000010:3.0:1775767588.530705:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e088. 00000100:00000001:3.0:1775767588.530706:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767588.530708:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fe680 x1862022294210176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767588.530714:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fe680 x1862022294210176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767588.530718:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767588.530718:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767588.530719:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767588.530720:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767588.530722:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fe680 x1862022294210176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767588.530726:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767588.530729:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fe680 x1862022294210176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767613 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767588.530734:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767588.530736:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767588.530737:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767588.530738:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767588.530739:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767588.530740:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767588.530742:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767588.530744:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767588.530745:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767588.530747:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fe680 x1862022294210176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767613 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767588.530752:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294210176:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767588.530754:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767588.530756:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fea00 x1862022294210304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767588.530760:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767588.530761:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767588.530763:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767588.530764:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767588.530765:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767588.530767:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294210304:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767588.530769:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767588.530770:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767588.530770:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767588.530771:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767588.530773:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b1000. 02000000:00000001:3.0:1775767588.530774:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767588.530777:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767588.530779:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b9a0. 00000400:00000010:3.0:1775767588.530781:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e088. 00000100:00000200:3.0:1775767588.530783:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294210304, portal 4 00000100:00000001:3.0:1775767588.530784:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767588.530785:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641207296 : -112966068344320 : ffff9942099fea00) 00000100:00000040:3.0:1775767588.530789:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fea00 x1862022294210304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767613 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767588.530793:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767588.530794:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767588.530795:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e660. 00000100:00000200:3.0:1775767588.530797:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294210304, offset 0 00000400:00000200:3.0:1775767588.530799:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767588.530801:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767588.530803:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767588.530805:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767588.530807:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767588.530809:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767588.530810:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767588.530812:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224816:224816:256:4294967295] 192.168.202.156@tcp LPNI seq info [224812:224812:8:4294967295] 00000400:00000200:3.0:1775767588.530814:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767588.530818:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767588.530821:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767588.530822:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8200. 00000800:00000200:3.0:1775767588.530825:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767588.530826:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775767588.530829:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767588.530831:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000100:3.0:1775767588.530833:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767588.530834:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8200. 00000400:00000200:3.0:1775767588.530836:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767588.530838:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767588.530840:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e660 00000400:00000010:3.0:1775767588.530841:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e660. 00000100:00000001:3.0:1775767588.530843:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767588.530845:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fea00 x1862022294210304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767613 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767588.530850:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767588.530852:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fea00 x1862022294210304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767613 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767588.530856:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767588.530857:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767588.530858:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767588.530860:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:3.0:1775767588.530862:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767588.530864:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299513378 00000100:00000001:3.0:1775767588.530866:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299514685 00000100:00000001:3.0:1775767588.530868:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299513378 00000100:00000001:3.0:1775767588.530870:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299514685 00000100:00000001:3.0:1775767588.530887:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767588.530890:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767588.530892:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767588.530897:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767588/real 1775767588] req@ffff9942099fea00 x1862022294210304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767588.530905:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767588.530907:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767588.530909:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b9a0. 00000400:00000200:3.0:1775767588.530912:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e088 00000400:00000010:3.0:1775767588.530914:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e088. 00000100:00000001:3.0:1775767588.530916:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767588.530919:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fea00 x1862022294210304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767588.530929:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fea00 x1862022294210304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767588.530935:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767588.530937:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767588.530938:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767588.530939:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767588.530942:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fea00 x1862022294210304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767588.530947:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767588.530949:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fea00 x1862022294210304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767613 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767588.530953:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767588.530954:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767588.530955:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767588.530956:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767588.530957:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767588.530961:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767588.530962:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767588.530964:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767588.530965:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767588.530967:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fea00 x1862022294210304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767613 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767588.530972:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294210304:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767588.530975:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767588.530977:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767588.530979:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fdc00 x1862022294210048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767613 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767588.530983:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767588.530984:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767588.530986:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b2800. 02000000:00000001:3.0:1775767588.530988:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767588.530989:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767588.530990:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:3.0:1775767588.530992:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767588.530993:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028000. 02000000:00000001:3.0:1775767588.530995:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767588.530996:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767588.530997:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fdc00. 00000100:00000001:3.0:1775767588.530999:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767588.531000:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767588.531001:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767588.531003:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fe680 x1862022294210176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767613 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767588.531008:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767588.531009:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767588.531010:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b2c00. 02000000:00000001:3.0:1775767588.531011:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767588.531012:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767588.531012:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:3.0:1775767588.531013:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767588.531014:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028400. 02000000:00000001:3.0:1775767588.531015:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767588.531016:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767588.531017:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fe680. 00000100:00000001:3.0:1775767588.531018:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767588.531019:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767588.531020:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767588.531021:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fea00 x1862022294210304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767613 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767588.531025:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767588.531026:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767588.531026:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b1000. 02000000:00000001:3.0:1775767588.531028:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767588.531028:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767588.531029:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:3.0:1775767588.531031:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767588.531032:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002a800. 02000000:00000001:3.0:1775767588.531033:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767588.531034:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767588.531035:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fea00. 00000100:00000001:3.0:1775767588.531036:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767588.531037:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767588.531038:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767588.531040:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767588.531041:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767588.531042:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767588.531043:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767588.531044:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767588.531046:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fca80 x1862022294210432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767588.531050:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767588.531052:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767588.531053:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767588.531054:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767588.531056:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767588.531057:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294210432:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767588.531059:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767588.531060:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767588.531061:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767588.531062:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767588.531064:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b1000. 02000000:00000001:3.0:1775767588.531066:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767588.531068:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767588.531070:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b9a0. 00000400:00000010:3.0:1775767588.531072:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e088. 00000100:00000200:3.0:1775767588.531074:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294210432, portal 4 00000100:00000001:3.0:1775767588.531075:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767588.531076:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641199232 : -112966068352384 : ffff9942099fca80) 00000100:00000040:3.0:1775767588.531080:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fca80 x1862022294210432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767613 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767588.531085:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767588.531087:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767588.531088:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e660. 00000100:00000200:3.0:1775767588.531090:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294210432, offset 0 00000400:00000200:3.0:1775767588.531092:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767588.531095:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767588.531098:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767588.531101:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767588.531104:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767588.531106:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767588.531109:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767588.531112:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224817:224817:256:4294967295] 192.168.202.156@tcp LPNI seq info [224813:224813:8:4294967295] 00000400:00000200:3.0:1775767588.531117:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767588.531123:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767588.531128:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767588.531131:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8200. 00000800:00000200:3.0:1775767588.531136:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767588.531139:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775767588.531143:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767588.531147:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000100:3.0:1775767588.531152:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767588.531171:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8200. 00000400:00000200:3.0:1775767588.531174:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767588.531177:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767588.531179:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e660 00000400:00000010:3.0:1775767588.531180:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e660. 00000100:00000001:3.0:1775767588.531182:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767588.531185:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fca80 x1862022294210432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767613 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767588.531192:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767588.531195:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fca80 x1862022294210432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767613 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767588.531209:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767588.531210:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767588.531214:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767588.531217:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:3.0:1775767588.531221:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767588.531223:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299513382 00000100:00000001:3.0:1775767588.531225:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299514685 00000100:00000001:3.0:1775767588.531228:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299513382 00000100:00000001:3.0:1775767588.531230:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299514685 00000100:00000001:3.0:1775767588.531233:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767588.531235:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767588.531237:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767588.531241:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767588/real 1775767588] req@ffff9942099fca80 x1862022294210432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767588.531250:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767588.531252:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767588.531253:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b9a0. 00000400:00000200:3.0:1775767588.531256:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e088 00000400:00000010:3.0:1775767588.531258:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e088. 00000100:00000001:3.0:1775767588.531260:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767588.531264:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fca80 x1862022294210432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767588.531270:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fca80 x1862022294210432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767588.531274:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767588.531275:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767588.531276:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767588.531277:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767588.531279:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fca80 x1862022294210432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767588.531284:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767588.531287:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fca80 x1862022294210432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767613 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767588.531291:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767588.531292:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767588.531293:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767588.531294:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767588.531295:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767588.531299:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767588.531300:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767588.531302:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767588.531304:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767588.531306:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fca80 x1862022294210432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767613 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767588.531310:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294210432:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767588.531313:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767588.531314:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767588.531316:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fca80 x1862022294210432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767613 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767588.531322:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767588.531323:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767588.531325:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b1000. 02000000:00000001:3.0:1775767588.531328:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767588.531329:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767588.531331:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:3.0:1775767588.531333:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767588.531334:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002bc00. 02000000:00000001:3.0:1775767588.531338:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767588.531339:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767588.531340:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fca80. 00000100:00000001:3.0:1775767588.531343:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767588.531344:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767588.531346:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767588.531350:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767588.531351:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767588.531353:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767588.531355:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767588.593336:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775767588.593341:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775767588.593344:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775767589.553232:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767589.553238:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767589.553240:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767589.553245:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767589.553249:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767589.553252:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767589.553254:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767589.553256:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767589.553261:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767590.577266:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767590.577274:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767590.577279:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767590.577288:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767590.577297:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767590.577303:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767590.577327:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767590.577331:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767590.577334:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767590.641374:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775767590.641381:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775767590.641385:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775767591.601254:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767591.601260:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767591.601263:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767591.601268:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767591.601272:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767591.601275:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767591.601279:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767591.601285:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767591.601291:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767592.625237:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767592.625244:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767592.625249:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767592.625257:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767592.625264:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767592.625270:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767592.625286:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767592.625289:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767592.625293:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767592.689430:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775767592.689433:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775767592.689436:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775767593.649285:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767593.649296:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767593.649299:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767593.649309:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767593.649322:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767593.649331:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767593.649339:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767593.649346:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767593.649350:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767593.649403:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767593.649413:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767593.649419:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767593.649432:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767593.649443:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767593.649448:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767593.649453:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767593.649461:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767593.649472:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767593.649719:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767593.649726:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767593.649733:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767593.649749:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dff100 x1862022294210560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767593.649774:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767593.649778:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767593.649784:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767593.649791:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767593.649794:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767593.649806:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294210560:192.168.202.156@tcp:250:kworker.0 00000100:00000001:3.0:1775767593.649819:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767593.649851:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767593.649856:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767593.649863:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767593.649873:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b1000. 02000000:00000001:3.0:1775767593.649886:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767593.649903:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767593.649909:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b9a0. 00000400:00000010:3.0:1775767593.649925:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e088. 00000100:00000200:3.0:1775767593.649942:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294210560, portal 25 00000100:00000001:3.0:1775767593.649957:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767593.649963:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611848960 : -112966097702656 : ffff994207dff100) 00000100:00000040:3.0:1775767593.649982:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dff100 x1862022294210560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767618 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767593.650010:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767593.650021:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767593.650025:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e660. 00000100:00000200:3.0:1775767593.650034:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294210560, offset 0 00000400:00000200:3.0:1775767593.650055:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767593.650070:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767593.650084:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767593.650098:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767593.650108:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767593.650121:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767593.650129:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767593.650150:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224818:224818:256:4294967295] 192.168.202.156@tcp LPNI seq info [224814:224814:8:4294967295] 00000400:00000200:3.0:1775767593.650188:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767593.650221:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767593.650239:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767593.650250:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8200. 00000800:00000200:3.0:1775767593.650273:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767593.650285:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775767593.650303:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767593.650315:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000100:3.0:1775767593.650333:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767593.650344:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8200. 00000400:00000200:3.0:1775767593.650355:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767593.650371:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767593.650384:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e660 00000400:00000010:3.0:1775767593.650391:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e660. 00000100:00000001:3.0:1775767593.650401:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767593.650417:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dff100 x1862022294210560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767618 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767593.650476:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767593.650486:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dff100 x1862022294210560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767618 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767593.650521:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767593.650527:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767593.650542:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767593.650558:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767593.650563:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299519806 00000100:00000010:3.0:1775767593.650571:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207981e40. 00000100:00000001:3.0:1775767593.650589:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1775767593.650596:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299519806 00000100:00000010:3.0:1775767593.650602:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207981900. 00000100:00000001:3.0:1775767593.650611:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1775767593.650619:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767593.650625:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767593.650632:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767593.650642:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767593/real 1775767593] req@ffff994207dff100 x1862022294210560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767593.650669:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767593.650673:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767593.650682:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b9a0. 00000400:00000200:3.0:1775767593.650690:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e088 00000400:00000010:3.0:1775767593.650697:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e088. 00000100:00000001:3.0:1775767593.650702:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767593.650713:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dff100 x1862022294210560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767593.650735:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dff100 x1862022294210560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767593.650748:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767593.650752:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767593.650757:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767593.650760:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767593.650774:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dff100 x1862022294210560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767593.650819:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767593.650845:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dff100 x1862022294210560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767618 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767593.650898:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767593.650904:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767593.650907:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767593.650911:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767593.650916:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767593.650919:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767593.650925:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767593.650936:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767593.650942:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767593.650953:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dff100 x1862022294210560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767618 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767593.650997:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294210560:192.168.202.156@tcp:250:kworker.0 00000100:00000001:3.0:1775767593.651006:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767593.651016:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767593.651030:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dff100 x1862022294210560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767618 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767593.651051:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767593.651055:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767593.651059:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b1000. 02000000:00000001:3.0:1775767593.651067:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767593.651070:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767593.651072:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:3.0:1775767593.651075:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767593.651080:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1fc00. 02000000:00000001:3.0:1775767593.651085:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767593.651091:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767593.651097:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dff100. 00000100:00000001:3.0:1775767593.651105:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767593.651111:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767593.651121:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767593.651132:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767593.651136:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767593.651148:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767593.651193:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767593.651197:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767593.651212:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfdc00 x1862022294210688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767593.651244:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767593.651250:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767593.651259:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767593.651266:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767593.651276:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767593.651287:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294210688:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767593.651301:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767593.651308:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767593.651312:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767593.651323:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767593.651331:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b1000. 02000000:00000001:3.0:1775767593.651339:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767593.651355:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767593.651367:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b9a0. 00000400:00000010:3.0:1775767593.651379:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e088. 00000100:00000200:3.0:1775767593.651393:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294210688, portal 10 00000100:00000001:3.0:1775767593.651402:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767593.651408:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611843584 : -112966097708032 : ffff994207dfdc00) 00000100:00000040:3.0:1775767593.651430:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfdc00 x1862022294210688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767618 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767593.651473:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767593.651487:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767593.651495:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e660. 00000100:00000200:3.0:1775767593.651505:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294210688, offset 0 00000400:00000200:3.0:1775767593.651524:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767593.651542:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767593.651561:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767593.651568:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767593.651573:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767593.651578:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767593.651582:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767593.651588:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224819:224819:256:4294967295] 192.168.202.156@tcp LPNI seq info [224815:224815:8:4294967295] 00000400:00000200:3.0:1775767593.651594:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767593.651603:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767593.651609:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767593.651615:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8200. 00000800:00000200:3.0:1775767593.651620:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767593.651623:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775767593.651629:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767593.651632:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000100:3.0:1775767593.651636:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767593.651638:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8200. 00000400:00000200:3.0:1775767593.651641:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767593.651645:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767593.651648:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e660 00000400:00000010:3.0:1775767593.651650:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e660. 00000100:00000001:3.0:1775767593.651653:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767593.651658:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfdc00 x1862022294210688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767618 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767593.651667:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767593.651671:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfdc00 x1862022294210688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767618 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767593.651678:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767593.651679:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767593.651683:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767593.651687:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:3.0:1775767593.651691:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767593.651693:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299518484 00000100:00000001:3.0:1775767593.651697:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299519806 00000100:00000001:3.0:1775767593.651699:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299518484 00000100:00000001:3.0:1775767593.651702:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299519806 00000100:00000001:3.0:1775767593.651704:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767593.651707:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767593.651709:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767593.651712:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767593/real 1775767593] req@ffff994207dfdc00 x1862022294210688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767593.651728:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767593.651730:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767593.651732:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b9a0. 00000400:00000200:3.0:1775767593.651735:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e088 00000400:00000010:3.0:1775767593.651736:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e088. 00000100:00000001:3.0:1775767593.651738:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767593.651742:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfdc00 x1862022294210688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767593.651750:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfdc00 x1862022294210688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767593.651758:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767593.651760:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767593.651762:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767593.651764:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767593.651768:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfdc00 x1862022294210688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767593.651775:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767593.651780:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfdc00 x1862022294210688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767618 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767593.651787:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767593.651790:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767593.651792:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767593.651793:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767593.651795:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767593.651799:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767593.651802:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767593.651807:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767593.651810:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767593.651815:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfdc00 x1862022294210688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767618 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767593.651840:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294210688:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767593.651846:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767593.651850:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfc700 x1862022294210816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767593.651858:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767593.651860:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767593.651863:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767593.651865:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767593.651868:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767593.651872:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294210816:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767593.651876:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767593.651878:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767593.651880:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767593.651882:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767593.651887:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b2c00. 02000000:00000001:3.0:1775767593.651890:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767593.651894:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767593.651896:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b9a0. 00000400:00000010:3.0:1775767593.651900:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e088. 00000100:00000200:3.0:1775767593.651904:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294210816, portal 10 00000100:00000001:3.0:1775767593.651906:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767593.651908:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611838208 : -112966097713408 : ffff994207dfc700) 00000100:00000040:3.0:1775767593.651914:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfc700 x1862022294210816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767618 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767593.651923:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767593.651925:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767593.651927:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e660. 00000100:00000200:3.0:1775767593.651930:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294210816, offset 0 00000400:00000200:3.0:1775767593.651935:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767593.651939:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767593.651943:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767593.651947:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767593.651950:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767593.651953:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767593.651956:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767593.651959:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224820:224820:256:4294967295] 192.168.202.156@tcp LPNI seq info [224816:224816:8:4294967295] 00000400:00000200:3.0:1775767593.651963:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767593.651970:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767593.651976:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767593.651980:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8200. 00000800:00000200:3.0:1775767593.651984:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767593.651987:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775767593.651992:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767593.651995:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000100:3.0:1775767593.651999:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767593.652001:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8200. 00000400:00000200:3.0:1775767593.652005:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767593.652008:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767593.652012:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e660 00000400:00000010:3.0:1775767593.652014:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e660. 00000100:00000001:3.0:1775767593.652016:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767593.652020:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfc700 x1862022294210816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767618 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767593.652028:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767593.652032:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfc700 x1862022294210816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767618 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767593.652038:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767593.652040:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767593.652043:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767593.652047:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:3.0:1775767593.652050:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767593.652053:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299518476 00000100:00000001:3.0:1775767593.652055:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299519806 00000100:00000001:3.0:1775767593.652057:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299518476 00000100:00000001:3.0:1775767593.652058:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299519806 00000100:00000001:3.0:1775767593.652060:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767593.652062:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767593.652063:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767593.652066:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767593/real 1775767593] req@ffff994207dfc700 x1862022294210816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767593.652071:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767593.652072:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767593.652073:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b9a0. 00000400:00000200:3.0:1775767593.652075:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e088 00000400:00000010:3.0:1775767593.652076:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e088. 00000100:00000001:3.0:1775767593.652078:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767593.652080:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfc700 x1862022294210816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767593.652085:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfc700 x1862022294210816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767593.652090:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767593.652090:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767593.652091:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767593.652092:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767593.652094:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfc700 x1862022294210816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767593.652098:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767593.652101:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfc700 x1862022294210816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767618 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767593.652105:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767593.652107:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767593.652108:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767593.652109:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767593.652110:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767593.652112:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767593.652114:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767593.652116:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767593.652117:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767593.652120:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfc700 x1862022294210816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767618 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767593.652125:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294210816:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767593.652127:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767593.652129:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767593.652131:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfdc00 x1862022294210688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767618 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767593.652135:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767593.652136:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767593.652137:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b1000. 02000000:00000001:3.0:1775767593.652139:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767593.652140:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767593.652141:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:3.0:1775767593.652142:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767593.652143:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1f800. 02000000:00000001:3.0:1775767593.652145:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767593.652146:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767593.652147:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfdc00. 00000100:00000001:3.0:1775767593.652149:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767593.652150:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767593.652151:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767593.652173:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfc700 x1862022294210816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767618 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767593.652182:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767593.652184:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767593.652185:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b2c00. 02000000:00000001:3.0:1775767593.652186:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767593.652187:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767593.652188:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:3.0:1775767593.652189:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767593.652190:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1d400. 02000000:00000001:3.0:1775767593.652192:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767593.652193:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767593.652194:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfc700. 00000100:00000001:3.0:1775767593.652196:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767593.652197:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767593.652198:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767593.652201:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767593.652202:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767593.652204:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767593.652205:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767593.652206:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767593.652209:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfe300 x1862022294210944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767593.652214:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767593.652215:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767593.652217:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767593.652218:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767593.652220:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767593.652222:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294210944:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767593.652224:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767593.652225:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767593.652227:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767593.652228:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767593.652230:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b2c00. 02000000:00000001:3.0:1775767593.652231:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767593.652234:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767593.652236:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b9a0. 00000400:00000010:3.0:1775767593.652239:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e088. 00000100:00000200:3.0:1775767593.652241:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294210944, portal 4 00000100:00000001:3.0:1775767593.652243:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767593.652243:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611845376 : -112966097706240 : ffff994207dfe300) 00000100:00000040:3.0:1775767593.652248:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfe300 x1862022294210944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767618 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767593.652252:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767593.652254:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767593.652255:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e660. 00000100:00000200:3.0:1775767593.652257:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294210944, offset 0 00000400:00000200:3.0:1775767593.652259:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767593.652262:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767593.652264:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767593.652266:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767593.652268:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767593.652270:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767593.652272:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767593.652274:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224821:224821:256:4294967295] 192.168.202.156@tcp LPNI seq info [224817:224817:8:4294967295] 00000400:00000200:3.0:1775767593.652277:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767593.652282:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767593.652285:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767593.652287:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8200. 00000800:00000200:3.0:1775767593.652290:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767593.652292:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775767593.652295:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767593.652297:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000100:3.0:1775767593.652300:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767593.652301:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8200. 00000400:00000200:3.0:1775767593.652303:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767593.652306:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767593.652308:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e660 00000400:00000010:3.0:1775767593.652309:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e660. 00000100:00000001:3.0:1775767593.652311:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767593.652313:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfe300 x1862022294210944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767618 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767593.652319:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767593.652320:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfe300 x1862022294210944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767618 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767593.652325:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767593.652326:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767593.652328:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767593.652330:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:3.0:1775767593.652332:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767593.652334:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299518500 00000100:00000001:3.0:1775767593.652336:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299519806 00000100:00000001:3.0:1775767593.652338:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299518500 00000100:00000001:3.0:1775767593.652339:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299519806 00000100:00000001:3.0:1775767593.652341:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767593.652342:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767593.652344:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767593.652346:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767593/real 1775767593] req@ffff994207dfe300 x1862022294210944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767593.652350:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767593.652351:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767593.652352:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b9a0. 00000400:00000200:3.0:1775767593.652353:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e088 00000400:00000010:3.0:1775767593.652354:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e088. 00000100:00000001:3.0:1775767593.652355:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767593.652357:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfe300 x1862022294210944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767593.652363:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfe300 x1862022294210944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767593.652367:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767593.652368:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767593.652369:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767593.652370:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767593.652373:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfe300 x1862022294210944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767593.652376:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767593.652379:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfe300 x1862022294210944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767618 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767593.652382:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767593.652384:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767593.652385:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767593.652386:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767593.652387:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767593.652390:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767593.652391:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767593.652393:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767593.652395:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767593.652397:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfe300 x1862022294210944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767618 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767593.652401:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294210944:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767593.652404:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767593.652407:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfed80 x1862022294211072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767593.652415:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767593.652417:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767593.652419:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767593.652421:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767593.652423:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767593.652426:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294211072:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767593.652430:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767593.652432:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767593.652433:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767593.652435:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767593.652438:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b1000. 02000000:00000001:3.0:1775767593.652440:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767593.652444:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767593.652446:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b9a0. 00000400:00000010:3.0:1775767593.652450:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e088. 00000100:00000200:3.0:1775767593.652452:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294211072, portal 4 00000100:00000001:3.0:1775767593.652455:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767593.652456:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611848064 : -112966097703552 : ffff994207dfed80) 00000100:00000040:3.0:1775767593.652462:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfed80 x1862022294211072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767618 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767593.652468:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767593.652469:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767593.652471:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e660. 00000100:00000200:3.0:1775767593.652473:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294211072, offset 0 00000400:00000200:3.0:1775767593.652475:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767593.652479:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767593.652481:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767593.652483:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767593.652485:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767593.652486:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767593.652488:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767593.652490:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224822:224822:256:4294967295] 192.168.202.156@tcp LPNI seq info [224818:224818:8:4294967295] 00000400:00000200:3.0:1775767593.652493:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767593.652497:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767593.652500:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767593.652503:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8200. 00000800:00000200:3.0:1775767593.652505:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767593.652507:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775767593.652510:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767593.652512:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000100:3.0:1775767593.652515:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767593.652516:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8200. 00000400:00000200:3.0:1775767593.652518:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767593.652520:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767593.652522:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e660 00000400:00000010:3.0:1775767593.652523:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e660. 00000100:00000001:3.0:1775767593.652525:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767593.652527:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfed80 x1862022294211072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767618 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767593.652532:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767593.652534:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfed80 x1862022294211072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767618 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767593.652538:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767593.652539:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767593.652541:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767593.652543:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:3.0:1775767593.652545:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767593.652546:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299518503 00000100:00000001:3.0:1775767593.652548:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299519806 00000100:00000001:3.0:1775767593.652549:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299518503 00000100:00000001:3.0:1775767593.652550:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299519806 00000100:00000001:3.0:1775767593.652552:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767593.652553:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767593.652554:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767593.652557:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767593/real 1775767593] req@ffff994207dfed80 x1862022294211072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767593.652562:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767593.652563:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767593.652565:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b9a0. 00000400:00000200:3.0:1775767593.652566:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e088 00000400:00000010:3.0:1775767593.652567:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e088. 00000100:00000001:3.0:1775767593.652568:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767593.652570:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfed80 x1862022294211072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767593.652575:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfed80 x1862022294211072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767593.652579:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767593.652580:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767593.652581:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767593.652582:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767593.652584:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfed80 x1862022294211072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767593.652588:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767593.652590:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfed80 x1862022294211072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767618 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767593.652594:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767593.652595:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767593.652596:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767593.652597:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767593.652598:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767593.652601:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767593.652603:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767593.652605:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767593.652606:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767593.652609:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfed80 x1862022294211072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767618 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767593.652614:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294211072:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767593.652617:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767593.652618:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767593.652619:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfe300 x1862022294210944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767618 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767593.652623:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767593.652624:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767593.652625:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b2c00. 02000000:00000001:3.0:1775767593.652626:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767593.652627:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767593.652628:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:3.0:1775767593.652629:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767593.652630:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1e400. 02000000:00000001:3.0:1775767593.652633:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767593.652634:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767593.652635:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfe300. 00000100:00000001:3.0:1775767593.652636:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767593.652637:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767593.652638:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767593.652640:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfed80 x1862022294211072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767618 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767593.652644:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767593.652645:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767593.652645:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b1000. 02000000:00000001:3.0:1775767593.652646:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767593.652647:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767593.652648:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:3.0:1775767593.652649:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767593.652650:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1d000. 02000000:00000001:3.0:1775767593.652651:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767593.652652:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767593.652653:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfed80. 00000100:00000001:3.0:1775767593.652655:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767593.652656:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767593.652657:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767593.652659:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767593.652660:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767593.652661:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767593.652663:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767594.673327:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767594.673337:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767594.673339:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767594.673345:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767594.673352:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767594.673357:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767594.673363:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767594.673370:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767594.673379:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767594.737408:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775767594.737412:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775767594.737415:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775767595.697220:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767595.697226:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767595.697228:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767595.697232:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767595.697236:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767595.697240:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767595.697243:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767595.697246:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767595.697248:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767596.721269:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767596.721275:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767596.721295:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767596.721314:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767596.721322:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767596.721334:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767596.721341:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767596.721361:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767596.721369:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767596.785377:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775767596.785381:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775767596.785385:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775767597.745206:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767597.745212:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767597.745214:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767597.745219:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767597.745224:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767597.745228:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767597.745231:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767597.745233:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767597.745236:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767598.769297:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767598.769306:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767598.769309:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767598.769315:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767598.769322:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767598.769328:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767598.769334:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767598.769342:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767598.769349:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767598.769401:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767598.769414:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775767598.769425:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767598.769430:0:36:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767598.769435:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767598.769444:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 4851 00000400:00000200:3.0:1775767598.769457:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767598.769467:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767598.769473:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767598.769479:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767598.769484:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767598.769488:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767598.769492:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767598.769498:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767598.769501:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:3.0:1775767598.769504:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767598.769508:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:3.0:1775767598.769511:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:3.0:1775767598.769514:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:3.0:1775767598.769519:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767598.769522:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767598.769525:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767598.769528:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767598.769532:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:3.0:1775767598.769536:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767598.769539:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767598.769543:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767598.769547:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767598.769567:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767598.769571:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775767598.769574:0:36:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767598.769582:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aad80. 00000020:00000040:3.0:1775767598.769587:0:36:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:3.0:1775767598.769591:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767598.769594:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767598.769597:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767598.769601:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767598.769607:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767598.769616:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b1000. 00000100:00000001:3.0:1775767598.769624:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767598.769635:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aad80 x1862022294211200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767598.769650:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767598.769655:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767598.769662:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000045ccf08] to pc [ptlrpcd_rcv+-1] req@ffff9942099aad80 x1862022294211200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767598.769693:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767598.769698:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767598.769703:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775767598.769706:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767598.769709:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767598.769714:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767598.769719:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4851 00000400:00000200:3.0:1775767598.769729:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767598.769739:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767598.769744:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767598.769755:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767598.769762:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767598.769771:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767598.769818:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767598.769825:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767598.769829:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:3.0:1775767598.769834:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767598.769838:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:3.0:1775767598.769847:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:3.0:1775767598.769855:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:3.0:1775767598.769884:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767598.769893:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767598.769899:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767598.769902:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767598.769910:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00080000:3.0:1775767598.769918:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767598.769936:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767598.769942:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767598.769950:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767598.769954:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767598.769957:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767598.769983:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775767598.769991:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767598.770002:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767598.770016:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aa680. 00000020:00000040:3.0:1775767598.770027:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:3.0:1775767598.770053:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767598.770058:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767598.770064:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767598.770070:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767598.770074:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767598.770089:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2c00. 00000100:00000001:3.0:1775767598.770096:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767598.770116:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aa680 x1862022294211328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767598.770152:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767598.770157:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767598.770167:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006cb1597d] to pc [ptlrpcd_rcv+-1] req@ffff9942099aa680 x1862022294211328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767598.770236:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767598.770244:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767598.770273:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775767598.770282:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767598.770286:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:2.1:1775767598.770295:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:3.0:1775767598.770297:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767598.770307:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4851 00000800:00000200:2.1:1775767598.770307:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000400:00000200:3.0:1775767598.770319:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767598.770331:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767598.770336:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767598.770346:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767598.770351:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767598.770360:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767598.770372:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767598.770378:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767598.770385:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:3.0:1775767598.770394:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767598.770398:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:3.0:1775767598.770404:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:3.0:1775767598.770413:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:3.0:1775767598.770428:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767598.770439:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767598.770451:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767598.770457:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767598.770463:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:3.0:1775767598.770471:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767598.770477:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767598.770484:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767598.770491:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767598.770501:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767598.770509:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767598.770515:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775767598.770523:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767598.770530:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767598.770535:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ab800. 00000020:00000040:3.0:1775767598.770540:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:3.0:1775767598.770551:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767598.770554:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767598.770563:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767598.770568:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767598.770577:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767598.770584:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2800. 00000100:00000001:3.0:1775767598.770590:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767598.770600:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ab800 x1862022294211456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767598.770626:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767598.770636:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767598.770649:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000089df80c0] to pc [ptlrpcd_rcv+-1] req@ffff9942099ab800 x1862022294211456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767598.770697:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767598.770705:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767598.770713:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775767598.770719:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767598.770722:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767598.770729:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767598.770739:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4851 00000400:00000200:3.0:1775767598.770750:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767598.770766:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767598.770771:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767598.770821:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767598.770829:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767598.770836:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767598.770848:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767598.770857:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767598.770880:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:3.0:1775767598.770884:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767598.770889:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:3.0:1775767598.770896:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:3.0:1775767598.770905:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:3.0:1775767598.770924:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767598.770936:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767598.770947:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767598.770960:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767598.770988:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:3.0:1775767598.771003:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767598.771013:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767598.771021:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767598.771028:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767598.771058:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767598.771062:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767598.771069:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:3.0:1775767598.771080:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767598.771092:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767598.771102:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aa300. 00000020:00000040:3.0:1775767598.771109:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:3.0:1775767598.771115:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767598.771119:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767598.771127:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767598.771134:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767598.771138:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767598.771152:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0000. 00000100:00000001:3.0:1775767598.771158:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767598.771167:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aa300 x1862022294211584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767598.771207:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767598.771214:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767598.771229:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f33bdf15] to pc [ptlrpcd_rcv+-1] req@ffff9942099aa300 x1862022294211584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767598.771312:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767598.771319:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767598.771344:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775767598.771352:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767598.771360:0:36:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767598.771371:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767598.771384:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4851 00000400:00000200:3.0:1775767598.771420:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767598.771434:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767598.771444:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767598.771455:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767598.771461:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767598.771472:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767598.771478:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767598.771489:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767598.771492:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:3.0:1775767598.771501:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767598.771508:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:3.0:1775767598.771515:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:3.0:1775767598.771529:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:3.0:1775767598.771546:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767598.771556:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767598.771562:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767598.771573:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767598.771581:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:3.0:1775767598.771590:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767598.771598:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767598.771623:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767598.771631:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767598.771643:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767598.771652:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767598.771658:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:3.0:1775767598.771688:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767598.771695:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767598.771709:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8a80. 00000020:00000040:3.0:1775767598.771716:0:36:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:3.0:1775767598.771727:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767598.771731:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767598.771742:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767598.771749:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767598.771757:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767598.771769:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0c00. 00000100:00000001:3.0:1775767598.771814:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767598.771826:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8a80 x1862022294211712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:2.1:1775767598.771872:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767598.771875:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000100:00000001:3.0:1775767598.771887:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767598.771898:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767598.771908:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006c3265af] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8a80 x1862022294211712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767598.771946:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767598.771956:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (4861) 00000800:00000200:2.1:1775767598.772911:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767598.772919:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775767598.773496:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767598.773499:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775767598.773751:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767598.773753:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000020:00000001:3.0:1775767598.833402:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775767598.833406:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775767598.833410:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775767599.793227:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767599.793235:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767599.793239:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767599.793246:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767599.793250:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767599.793255:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767599.793258:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767599.793261:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767599.793265:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767600.817229:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767600.817240:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767600.817242:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767600.817250:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767600.817262:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767600.817269:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767600.817276:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767600.817281:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767600.817285:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767600.881366:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775767600.881370:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775767600.881373:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775767601.841226:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767601.841234:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767601.841237:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767601.841242:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767601.841249:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767601.841254:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767601.841261:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767601.841266:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767601.841272:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767602.865206:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767602.865213:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767602.865216:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767602.865221:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767602.865225:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767602.865229:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767602.865233:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767602.865237:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767602.865242:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767602.929316:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775767602.929319:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775767602.929322:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775767603.889269:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767603.889278:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767603.889285:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767603.889296:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767603.889307:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767603.889314:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767603.889327:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767603.889334:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767603.889340:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775767603.889818:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767603.889823:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775767603.890497:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767603.890498:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775767603.890684:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767603.890685:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775767603.890887:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767603.890890:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775767603.891269:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767603.891271:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000100:00000001:3.0:1775767604.913224:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767604.913232:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767604.913237:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767604.913245:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767604.913252:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767604.913257:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767604.913263:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767604.913267:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767604.913270:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767604.977352:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775767604.977361:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775767604.977364:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775767605.937240:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767605.937248:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767605.937252:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767605.937259:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767605.937265:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767605.937270:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767605.937283:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767605.937287:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767605.937291:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767607.025288:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775767607.025291:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775767607.025294:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767609.009263:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767609.009279:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775767609.009285:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767609.009291:0:36:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767609.009300:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767609.009310:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 4861 00000400:00000200:3.0:1775767609.009332:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767609.009342:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767609.009347:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767609.009351:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767609.009355:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767609.009359:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767609.009362:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767609.009367:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767609.009370:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:3.0:1775767609.009372:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767609.009374:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:3.0:1775767609.009376:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:3.0:1775767609.009378:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:3.0:1775767609.009380:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767609.009382:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767609.009383:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767609.009385:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767609.009387:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:3.0:1775767609.009389:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767609.009391:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767609.009393:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767609.009395:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767609.009397:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767609.009398:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775767609.009400:0:36:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767609.009409:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8a80. 00000020:00000040:3.0:1775767609.009411:0:36:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:3.0:1775767609.009414:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767609.009415:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767609.009417:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767609.009420:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767609.009423:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767609.009431:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b3400. 00000100:00000001:3.0:1775767609.009437:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767609.009444:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8a80 x1862022294212480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767609.009452:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767609.009455:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767609.009459:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006c3265af] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8a80 x1862022294212480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767609.009476:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767609.009479:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767609.009482:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775767609.009483:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767609.009484:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767609.009487:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767609.009489:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4861 00000400:00000200:3.0:1775767609.009493:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767609.009496:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767609.009498:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767609.009501:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767609.009504:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767609.009508:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767609.009511:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767609.009514:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767609.009516:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:3.0:1775767609.009517:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767609.009520:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:3.0:1775767609.009523:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:3.0:1775767609.009528:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:3.0:1775767609.009533:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767609.009538:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767609.009542:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767609.009547:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767609.009552:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00080000:3.0:1775767609.009557:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767609.009562:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767609.009567:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767609.009576:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767609.009582:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767609.009585:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767609.009591:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775767609.009596:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767609.009601:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767609.009607:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aa300. 00000020:00000040:3.0:1775767609.009610:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:3.0:1775767609.009617:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767609.009620:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767609.009623:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767609.009627:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767609.009631:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767609.009637:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2000. 00000100:00000001:3.0:1775767609.009654:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767609.009666:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aa300 x1862022294212608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767609.009716:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767609.009720:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767609.009729:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f33bdf15] to pc [ptlrpcd_rcv+-1] req@ffff9942099aa300 x1862022294212608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767609.009749:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767609.009754:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767609.009761:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775767609.009765:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767609.009768:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767609.009774:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767609.009781:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4861 00000400:00000200:3.0:1775767609.009792:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767609.009797:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767609.009800:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767609.009807:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767609.009810:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767609.009814:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767609.009818:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767609.009821:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767609.009822:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:3.0:1775767609.009824:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767609.009827:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:3.0:1775767609.009831:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:3.0:1775767609.009834:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:3.0:1775767609.009836:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767609.009838:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767609.009841:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767609.009843:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767609.009845:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:3.0:1775767609.009850:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767609.009853:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767609.009856:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767609.009860:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767609.009864:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775767609.009864:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:3.0:1775767609.009867:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000800:00000200:2.1:1775767609.009869:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000008:00100000:3.0:1775767609.009870:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775767609.009876:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767609.009878:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767609.009882:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ab800. 00000020:00000040:3.0:1775767609.009885:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:3.0:1775767609.009887:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767609.009889:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767609.009893:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767609.009895:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767609.009899:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767609.009906:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0400. 00000100:00000001:3.0:1775767609.009910:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767609.009917:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ab800 x1862022294212736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767609.009928:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767609.009932:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767609.009939:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000089df80c0] to pc [ptlrpcd_rcv+-1] req@ffff9942099ab800 x1862022294212736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767609.009949:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767609.009953:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767609.009960:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775767609.009962:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767609.009964:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767609.009967:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767609.009971:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4861 00000400:00000200:3.0:1775767609.009975:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767609.009981:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767609.009984:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767609.009989:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767609.009993:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767609.009995:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767609.009997:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767609.010002:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767609.010002:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:3.0:1775767609.010003:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767609.010005:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:3.0:1775767609.010006:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:3.0:1775767609.010008:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:3.0:1775767609.010011:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767609.010014:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767609.010016:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767609.010018:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767609.010021:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:3.0:1775767609.010024:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767609.010026:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767609.010029:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767609.010032:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767609.010034:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767609.010037:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767609.010040:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:3.0:1775767609.010042:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767609.010044:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767609.010049:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aa680. 00000020:00000040:3.0:1775767609.010053:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:3.0:1775767609.010056:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767609.010058:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767609.010060:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767609.010061:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767609.010062:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767609.010066:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0c00. 00000100:00000001:3.0:1775767609.010071:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767609.010079:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aa680 x1862022294212864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767609.010103:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767609.010105:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767609.010110:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006cb1597d] to pc [ptlrpcd_rcv+-1] req@ffff9942099aa680 x1862022294212864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767609.010135:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767609.010139:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767609.010145:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775767609.010148:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767609.010151:0:36:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767609.010189:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767609.010194:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4861 00000400:00000200:3.0:1775767609.010201:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767609.010206:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767609.010210:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767609.010214:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767609.010218:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767609.010222:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767609.010227:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767609.010232:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767609.010235:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:3.0:1775767609.010239:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767609.010242:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:3.0:1775767609.010246:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:3.0:1775767609.010251:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:3.0:1775767609.010257:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767609.010262:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767609.010267:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767609.010272:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767609.010277:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:3.0:1775767609.010284:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767609.010292:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767609.010298:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767609.010303:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767609.010307:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767609.010312:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767609.010316:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:3.0:1775767609.010320:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767609.010324:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767609.010329:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aad80. 00000020:00000040:3.0:1775767609.010333:0:36:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:3.0:1775767609.010336:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767609.010338:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767609.010341:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767609.010344:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767609.010346:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767609.010351:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0000. 00000100:00000001:3.0:1775767609.010357:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767609.010362:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aad80 x1862022294212992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767609.010376:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767609.010379:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767609.010385:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000045ccf08] to pc [ptlrpcd_rcv+-1] req@ffff9942099aad80 x1862022294212992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767609.010398:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767609.010403:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (4871) 00000800:00000200:2.1:1775767609.010512:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767609.010513:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775767609.010794:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767609.010797:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775767609.011010:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767609.011013:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775767609.011268:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767609.011270:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000020:00000001:3.0:1775767609.073298:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775767609.073301:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775767609.073304:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1775767611.121314:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775767611.121318:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775767611.121320:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775767612.081370:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767612.081375:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767612.081378:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767612.081384:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767612.081390:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767612.081395:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767612.081400:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767612.081402:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767612.081405:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767613.105241:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767613.105249:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767613.105252:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767613.105261:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767613.105266:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767613.105270:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767613.105275:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767613.105279:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767613.105284:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767613.169455:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775767613.169459:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775767613.169461:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775767614.129291:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767614.129304:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767614.129308:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767614.129316:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767614.129325:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767614.129333:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767614.129356:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767614.129361:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767614.129374:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775767614.129961:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767614.129967:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775767614.130702:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767614.130704:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775767614.130996:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767614.130998:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775767614.131290:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767614.131292:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775767614.131504:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767614.131506:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000100:00000001:3.0:1775767615.153237:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767615.153242:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767615.153246:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767615.153251:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767615.153255:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767615.153263:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767615.153267:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767615.153270:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767615.153274:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767615.217347:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775767615.217352:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775767615.217355:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775767616.177239:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767616.177259:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767616.177264:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767616.177272:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767616.177279:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767616.177284:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767616.177317:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767616.177326:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767616.177331:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767617.201211:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767617.201218:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767617.201221:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767617.201227:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767617.201232:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767617.201235:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767617.201239:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767617.201242:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767617.201248:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767617.265298:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775767617.265302:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775767617.265306:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775767618.225217:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767618.225222:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767618.225226:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767618.225231:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767618.225236:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767618.225242:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767618.225264:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767618.225266:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767618.225268:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767619.249231:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767619.249238:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767619.249241:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767619.249249:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767619.249258:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767619.249265:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767619.249287:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767619.249293:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767619.249299:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775767619.249777:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767619.249782:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775767619.250437:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767619.250439:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775767619.250703:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767619.250705:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775767619.251023:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767619.251024:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775767619.251260:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767619.251261:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000020:00000001:3.0:1775767619.313369:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775767619.313373:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775767619.313377:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775767620.273265:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767620.273277:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767620.273280:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767620.273287:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767620.273301:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767620.273311:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767620.273321:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767620.273328:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767620.273334:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767621.297224:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767621.297231:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767621.297235:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767621.297242:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767621.297247:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767621.297251:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767621.297267:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767621.297270:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767621.297273:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767621.361396:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775767621.361402:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775767621.361407:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775767622.321233:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767622.321241:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767622.321245:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767622.321250:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767622.321255:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767622.321261:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767622.321268:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767622.321271:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767622.321275:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767623.345229:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767623.345236:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767623.345240:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767623.345245:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767623.345248:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767623.345252:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767623.345263:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767623.345266:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767623.345268:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767623.409310:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775767623.409313:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775767623.409316:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00000001:3.0:1775767624.369432:0:13471:0:(rw.c:1481:ll_writepages()) Process entered 00000080:00000001:3.0:1775767624.369436:0:13471:0:(file.c:5250:cl_sync_file_range()) Process entered 00000020:00000001:3.0:1775767624.369438:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:3.0:1775767624.369446:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff99420693e000. 00000020:00000010:3.0:1775767624.369450:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcec00. 00000020:00000010:3.0:1775767624.369465:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210dcf600. 00000020:00000010:3.0:1775767624.369468:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942079812a0. 00000008:00000010:3.0:1775767624.369474:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994208b50000. 00020000:00000010:3.0:1775767624.369477:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994218000300. 00008000:00000010:3.0:1775767624.369481:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994207e08e20. 00000080:00000010:3.0:1775767624.369486:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99420541b1f8. 00000080:00000010:3.0:1775767624.369488:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077c0248. 00000020:00000010:3.0:1775767624.369491:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcfe00. 00000008:00000010:3.0:1775767624.369494:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942086f8420. 00020000:00000010:3.0:1775767624.369496:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994207e8e400. 00008000:00000010:3.0:1775767624.369499:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994209270a90. 00000080:00000010:3.0:1775767624.369502:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942090e33c0. 00000020:00000001:3.0:1775767624.369504:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107590086664 : -112966119464952 : ffff99420693e008) 00000020:00001000:3.0:1775767624.369506:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff99420693e000 00000020:00000001:3.0:1775767624.369510:0:13471:0:(cl_object.c:151:cl_object_top()) ffff99420791a5f8 -> ffff99420791a5f8 00000020:00000001:3.0:1775767624.369512:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:3.0:1775767624.369514:0:13471:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:3.0:1775767624.369515:0:13471:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x10176:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:3.0:1775767624.369518:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:3.0:1775767624.369519:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:3.0:1775767624.369521:0:13471:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:3.0:1775767624.369524:0:13471:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x10176:0x0]io ffff9942077c02a8 type 6 ignore/verify layout 1/0 00020000:00000002:3.0:1775767624.369555:0:13471:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207dfb6c8) owner 0000000000000000/ffff994220080000 00020000:00000001:3.0:1775767624.369560:0:13471:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:3.0:1775767624.369561:0:13471:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:3.0:1775767624.369563:0:13471:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00020000:00000001:3.0:1775767624.369565:0:13471:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767624.369568:0:13471:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:3.0:1775767624.369571:0:13471:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:3.0:1775767624.369573:0:13471:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767624.369575:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:3.0:1775767624.369576:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:3.0:1775767624.369577:0:13471:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00020000:00000002:3.0:1775767624.369579:0:13471:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207dfb6c8) owner 0000000000000000/ffff994220080000 00000020:00000001:3.0:1775767624.369582:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767624.369584:0:13471:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:3.0:1775767624.369586:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00020000:00000001:3.0:1775767624.369588:0:13471:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00020000:00200000:3.0:1775767624.369591:0:13471:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00000002:3.0:1775767624.369595:0:13471:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:3.0:1775767624.369599:0:13471:0:(lov_io.c:155:lov_sub_get()) Process entered 00020000:00000001:3.0:1775767624.369600:0:13471:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000020:00000001:3.0:1775767624.369602:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:3.0:1775767624.369605:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff99420693ecf0. 00000020:00000010:3.0:1775767624.369608:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcf800. 00000020:00000010:3.0:1775767624.369612:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210dcf000. 00000020:00000010:3.0:1775767624.369615:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207981f60. 00000008:00000010:3.0:1775767624.369618:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994208b51650. 00020000:00000010:3.0:1775767624.369622:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994218000000. 00008000:00000010:3.0:1775767624.369624:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994207e0ade8. 00000080:00000010:3.0:1775767624.369628:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99420541b0a8. 00000080:00000010:3.0:1775767624.369630:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077c1da8. 00000020:00000010:3.0:1775767624.369634:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcf200. 00000008:00000010:3.0:1775767624.369637:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942086f8160. 00020000:00000010:3.0:1775767624.369639:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994207e8ed00. 00008000:00000010:3.0:1775767624.369642:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994209270ef0. 00000080:00000010:3.0:1775767624.369645:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942090e3960. 00000020:00000001:3.0:1775767624.369647:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107590089976 : -112966119461640 : ffff99420693ecf8) 00000020:00001000:3.0:1775767624.369650:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff99420693ecf0 00000020:00000001:3.0:1775767624.369652:0:13471:0:(cl_object.c:151:cl_object_top()) ffff99420791ae90 -> ffff99420791a5f8 00000020:00000001:3.0:1775767624.369654:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:3.0:1775767624.369657:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:3.0:1775767624.369659:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:3.0:1775767624.369660:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767624.369662:0:13471:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767624.369665:0:13471:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107612435560 : -112966097116056 : ffff994207e8e468) 00000020:00000001:3.0:1775767624.369667:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000020:00000001:3.0:1775767624.369670:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767624.369673:0:13471:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767624.369675:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767624.369677:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00020000:00000001:3.0:1775767624.369679:0:13471:0:(lov_io.c:1108:lov_io_lock()) Process entered 00020000:00000001:3.0:1775767624.369680:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775767624.369682:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:3.0:1775767624.369683:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:3.0:1775767624.369685:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767624.369687:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767624.369690:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767624.369691:0:13471:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767624.369693:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:3.0:1775767624.369694:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767624.369696:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767624.369698:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00020000:00000001:3.0:1775767624.369700:0:13471:0:(lov_io.c:1114:lov_io_start()) Process entered 00020000:00000001:3.0:1775767624.369702:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775767624.369703:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00000008:00000001:3.0:1775767624.369705:0:13471:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000008:00000001:3.0:1775767624.369708:0:13471:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000008:00000001:3.0:1775767624.369713:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:3.0:1775767624.369715:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:3.0:1775767624.369717:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1775767624.369719:0:13471:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000008:00000001:3.0:1775767624.369721:0:13471:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000008:00000001:3.0:1775767624.369723:0:13471:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107754940544 : -112965954611072 : ffff994210675880) 00000008:00000020:3.0:1775767624.369726:0:13471:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994210675880 ready 0|-|- wr 1|-|+ rd 0|- 10 in flight 00000008:00000001:3.0:1775767624.369730:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:3.0:1775767624.369731:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:3.0:1775767624.369733:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1775767624.369735:0:13471:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000008:00000020:3.0:1775767624.369737:0:13471:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994218232fc0 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000008:00000001:3.0:1775767624.369742:0:13471:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1775767624.369744:0:13471:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767624.369746:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767624.369747:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767624.369749:0:13471:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767624.369751:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767624.369753:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00020000:00000001:3.0:1775767624.369756:0:13471:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00020000:00000001:3.0:1775767624.369758:0:13471:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000020:00000001:3.0:1775767624.369759:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000001:3.0:1775767624.369760:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00020000:00000001:3.0:1775767624.369762:0:13471:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767624.369763:0:13471:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000020:00000001:3.0:1775767624.369765:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00000020:00000001:3.0:1775767624.369766:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00020000:00000001:3.0:1775767624.369768:0:13471:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00020000:00000001:3.0:1775767624.369770:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775767624.369771:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000020:00000001:3.0:1775767624.369772:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:3.0:1775767624.369774:0:13471:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767624.369776:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767624.369777:0:13471:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000020:00000001:3.0:1775767624.369779:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:3.0:1775767624.369780:0:13471:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000020:00000001:3.0:1775767624.369782:0:13471:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000001:3.0:1775767624.369783:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:3.0:1775767624.369785:0:13471:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00020000:00000001:3.0:1775767624.369786:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775767624.369788:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000001:3.0:1775767624.369789:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00020000:00000001:3.0:1775767624.369791:0:13471:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767624.369793:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767624.369795:0:13471:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000020:00000001:3.0:1775767624.369797:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000020:00000001:3.0:1775767624.369799:0:13471:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767624.369801:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:3.0:1775767624.369803:0:13471:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:3.0:1775767624.369805:0:13471:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:3.0:1775767624.369806:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:3.0:1775767624.369809:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:3.0:1775767624.369812:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff99420693ecf0 00000020:00000010:3.0:1775767624.369817:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210dcf000. 00000020:00000010:3.0:1775767624.369824:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207981f60. 00000008:00000010:3.0:1775767624.369831:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994208b51650. 00020000:00000010:3.0:1775767624.369838:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994218000000. 00008000:00000010:3.0:1775767624.369843:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994207e0ade8. 00000080:00000010:3.0:1775767624.369849:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99420541b0a8. 00000080:00000010:3.0:1775767624.369854:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077c1da8. 00000020:00000010:3.0:1775767624.369859:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcf800. 00000008:00000010:3.0:1775767624.369864:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942086f8160. 00020000:00000010:3.0:1775767624.369870:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994207e8ed00. 00008000:00000010:3.0:1775767624.369875:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994209270ef0. 00000080:00000010:3.0:1775767624.369880:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942090e3960. 00000020:00000010:3.0:1775767624.369884:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcf200. 00000020:00000010:3.0:1775767624.369887:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff99420693ecf0. 00020000:00000001:3.0:1775767624.369892:0:13471:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:3.0:1775767624.369895:0:13471:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:3.0:1775767624.369899:0:13471:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000080:00200000:3.0:1775767624.369902:0:13471:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x10176:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:3.0:1775767624.369907:0:13471:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:3.0:1775767624.369909:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:3.0:1775767624.369913:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff99420693e000 00000020:00000010:3.0:1775767624.369915:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210dcf600. 00000020:00000010:3.0:1775767624.369918:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942079812a0. 00000008:00000010:3.0:1775767624.369924:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994208b50000. 00020000:00000010:3.0:1775767624.369944:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994218000300. 00008000:00000010:3.0:1775767624.369949:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994207e08e20. 00000080:00000010:3.0:1775767624.369953:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99420541b1f8. 00000080:00000010:3.0:1775767624.369956:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077c0248. 00000020:00000010:3.0:1775767624.369961:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcec00. 00000008:00000010:3.0:1775767624.369964:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942086f8420. 00020000:00000010:3.0:1775767624.369968:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994207e8e400. 00008000:00000010:3.0:1775767624.369974:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994209270a90. 00000080:00000010:3.0:1775767624.369977:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942090e33c0. 00000020:00000010:3.0:1775767624.369982:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcfe00. 00000020:00000010:3.0:1775767624.369986:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff99420693e000. 00000080:00000001:3.0:1775767624.369991:0:13471:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:3.0:1775767624.369996:0:13471:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775767624.370099:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767624.370105:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775767624.370724:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767624.370726:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775767624.370905:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767624.370907:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775767624.371107:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767624.371110:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775767624.371504:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767624.371507:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000020:00000001:3.0:1775767625.457395:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775767625.457399:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775767625.457401:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775767626.417288:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767626.417293:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767626.417296:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767626.417303:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767626.417309:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767626.417312:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767626.417316:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767626.417319:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767626.417322:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767627.441437:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767627.441441:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767627.441443:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767627.441446:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767627.441449:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767627.441452:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767627.441455:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767627.441457:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767627.441459:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767627.505334:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775767627.505339:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775767627.505343:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775767628.465293:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767628.465302:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767628.465306:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767628.465314:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767628.465325:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767628.465333:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767628.465344:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767628.465351:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767628.465355:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767629.489253:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767629.489257:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767629.489261:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767629.489266:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767629.489270:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767629.489274:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767629.489280:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767629.489284:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767629.489290:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767629.489558:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767629.489563:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767629.489567:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767629.489577:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207889880 x1862022294215040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767629.489602:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767629.489606:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767629.489609:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767629.489614:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767629.489618:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767629.489627:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207889880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294215040:192.168.202.156@tcp:250:kworker.0 00000100:00000001:3.0:1775767629.489636:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767629.489643:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767629.489647:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767629.489650:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767629.489656:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b2800. 02000000:00000001:3.0:1775767629.489661:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767629.489673:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767629.489680:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b9a0. 00000400:00000010:3.0:1775767629.489688:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e088. 00000100:00000200:3.0:1775767629.489697:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294215040, portal 25 00000100:00000001:3.0:1775767629.489702:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767629.489706:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606124672 : -112966103426944 : ffff994207889880) 00000100:00000040:3.0:1775767629.489716:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207889880 x1862022294215040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767654 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767629.489736:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767629.489745:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767629.489752:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e660. 00000100:00000200:3.0:1775767629.489757:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294215040, offset 0 00000400:00000200:3.0:1775767629.489768:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767629.489779:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767629.489789:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767629.489799:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767629.489807:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767629.489815:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767629.489824:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767629.489835:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224853:224853:256:4294967295] 192.168.202.156@tcp LPNI seq info [224849:224849:8:4294967295] 00000400:00000200:3.0:1775767629.489846:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767629.489867:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767629.489884:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767629.489892:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8200. 00000800:00000200:3.0:1775767629.489906:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767629.489914:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1775767629.489930:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767629.489938:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000100:3.0:1775767629.489954:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767629.489958:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8200. 00000400:00000200:3.0:1775767629.489967:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767629.489980:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767629.489990:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e660 00000400:00000010:3.0:1775767629.489996:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e660. 00000100:00000001:3.0:1775767629.490004:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767629.490013:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207889880 x1862022294215040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767654 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767629.490050:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767629.490058:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207889880 x1862022294215040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767654 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767629.490085:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767629.490091:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767629.490100:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767629.490111:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767629.490119:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299555645 00000100:00000010:3.0:1775767629.490129:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942079812a0. 00000100:00000001:3.0:1775767629.490141:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1775767629.490151:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299555645 00000100:00000010:3.0:1775767629.490177:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207981f60. 00000100:00000001:3.0:1775767629.490183:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1775767629.490189:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767629.490192:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767629.490196:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767629.490205:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767629/real 1775767629] req@ffff994207889880 x1862022294215040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767629.490232:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767629.490236:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767629.490241:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b9a0. 00000400:00000200:3.0:1775767629.490246:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e088 00000400:00000010:3.0:1775767629.490251:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e088. 00000100:00000001:3.0:1775767629.490258:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767629.490263:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207889880 x1862022294215040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767629.490296:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207889880 x1862022294215040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767629.490315:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767629.490318:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767629.490324:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767629.490327:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767629.490333:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207889880 x1862022294215040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767629.490354:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767629.490363:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207889880 x1862022294215040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767654 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767629.490383:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767629.490389:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767629.490392:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767629.490394:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767629.490398:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767629.490400:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767629.490404:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767629.490414:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767629.490421:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767629.490429:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207889880 x1862022294215040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767654 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767629.490466:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207889880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294215040:192.168.202.156@tcp:250:kworker.0 00000100:00000001:3.0:1775767629.490490:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767629.490495:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767629.490501:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207889880 x1862022294215040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767654 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767629.490527:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767629.490532:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767629.490536:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b2800. 02000000:00000001:3.0:1775767629.490542:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767629.490546:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767629.490550:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:3.0:1775767629.490555:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767629.490559:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543c800. 02000000:00000001:3.0:1775767629.490565:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767629.490569:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767629.490572:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207889880. 00000100:00000001:3.0:1775767629.490579:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767629.490583:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767629.490586:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767629.490595:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767629.490598:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767629.490604:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767629.490607:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767629.490611:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767629.490617:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff99420788b100 x1862022294215168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767629.490642:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767629.490645:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767629.490652:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767629.490655:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767629.490661:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767629.490667:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff99420788b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294215168:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767629.490674:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767629.490679:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767629.490681:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767629.490686:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767629.490692:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b2800. 02000000:00000001:3.0:1775767629.490696:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767629.490705:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767629.490709:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b9a0. 00000400:00000010:3.0:1775767629.490714:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e088. 00000100:00000200:3.0:1775767629.490719:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294215168, portal 10 00000100:00000001:3.0:1775767629.490724:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767629.490727:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606130944 : -112966103420672 : ffff99420788b100) 00000100:00000040:3.0:1775767629.490740:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff99420788b100 x1862022294215168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767654 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767629.490759:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767629.490767:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767629.490772:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e660. 00000100:00000200:3.0:1775767629.490778:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294215168, offset 0 00000400:00000200:3.0:1775767629.490786:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767629.490793:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767629.490799:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767629.490803:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767629.490809:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767629.490814:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767629.490818:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767629.490824:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224854:224854:256:4294967295] 192.168.202.156@tcp LPNI seq info [224850:224850:8:4294967295] 00000400:00000200:3.0:1775767629.490831:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767629.490842:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767629.490853:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767629.490859:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8200. 00000800:00000200:3.0:1775767629.490865:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767629.490869:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1775767629.490879:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767629.490883:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000100:3.0:1775767629.490893:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767629.490895:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8200. 00000400:00000200:3.0:1775767629.490899:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767629.490908:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767629.490914:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e660 00000400:00000010:3.0:1775767629.490917:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e660. 00000100:00000001:3.0:1775767629.490922:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767629.490925:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff99420788b100 x1862022294215168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767654 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767629.490942:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767629.490946:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff99420788b100 x1862022294215168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767654 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767629.490972:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767629.490976:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767629.490980:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767629.490985:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:3.0:1775767629.490991:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767629.490994:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299554323 00000100:00000001:3.0:1775767629.490999:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299555645 00000100:00000001:3.0:1775767629.491004:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299554323 00000100:00000001:3.0:1775767629.491009:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299555645 00000100:00000001:3.0:1775767629.491013:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767629.491018:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767629.491023:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767629.491025:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767629/real 1775767629] req@ffff99420788b100 x1862022294215168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767629.491049:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767629.491052:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767629.491056:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b9a0. 00000400:00000200:3.0:1775767629.491061:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e088 00000400:00000010:3.0:1775767629.491064:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e088. 00000100:00000001:3.0:1775767629.491067:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767629.491073:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff99420788b100 x1862022294215168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767629.491103:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff99420788b100 x1862022294215168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767629.491127:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767629.491129:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767629.491131:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767629.491134:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767629.491138:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff99420788b100 x1862022294215168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767629.491149:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767629.491168:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff99420788b100 x1862022294215168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767654 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767629.491174:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767629.491175:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767629.491177:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767629.491178:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767629.491180:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767629.491182:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767629.491187:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767629.491192:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767629.491195:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767629.491202:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff99420788b100 x1862022294215168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767654 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767629.491215:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff99420788b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294215168:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767629.491221:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767629.491226:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207888e00 x1862022294215296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767629.491241:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767629.491244:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767629.491247:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767629.491250:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767629.491255:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767629.491259:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207888e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294215296:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767629.491266:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767629.491269:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767629.491273:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767629.491276:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767629.491281:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b2c00. 02000000:00000001:3.0:1775767629.491285:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767629.491292:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767629.491296:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b9a0. 00000400:00000010:3.0:1775767629.491301:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e088. 00000100:00000200:3.0:1775767629.491307:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294215296, portal 10 00000100:00000001:3.0:1775767629.491311:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767629.491313:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606121984 : -112966103429632 : ffff994207888e00) 00000100:00000040:3.0:1775767629.491320:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207888e00 x1862022294215296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767654 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767629.491340:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767629.491345:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767629.491349:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e660. 00000100:00000200:3.0:1775767629.491354:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294215296, offset 0 00000400:00000200:3.0:1775767629.491364:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767629.491370:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767629.491376:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767629.491381:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767629.491387:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767629.491392:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767629.491396:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767629.491403:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224855:224855:256:4294967295] 192.168.202.156@tcp LPNI seq info [224851:224851:8:4294967295] 00000400:00000200:3.0:1775767629.491411:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767629.491423:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767629.491431:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767629.491435:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8200. 00000800:00000200:3.0:1775767629.491443:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767629.491448:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1775767629.491455:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767629.491458:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000100:3.0:1775767629.491465:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767629.491467:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8200. 00000400:00000200:3.0:1775767629.491471:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767629.491487:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767629.491490:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e660 00000400:00000010:3.0:1775767629.491491:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e660. 00000100:00000001:3.0:1775767629.491494:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767629.491496:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207888e00 x1862022294215296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767654 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767629.491508:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767629.491513:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207888e00 x1862022294215296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767654 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767629.491527:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767629.491529:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767629.491533:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767629.491537:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:3.0:1775767629.491544:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767629.491548:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299554316 00000100:00000001:3.0:1775767629.491551:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299555645 00000100:00000001:3.0:1775767629.491554:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299554316 00000100:00000001:3.0:1775767629.491557:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299555645 00000100:00000001:3.0:1775767629.491561:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767629.491564:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767629.491567:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767629.491573:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767629/real 1775767629] req@ffff994207888e00 x1862022294215296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767629.491589:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767629.491591:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767629.491594:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b9a0. 00000400:00000200:3.0:1775767629.491597:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e088 00000400:00000010:3.0:1775767629.491601:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e088. 00000100:00000001:3.0:1775767629.491607:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767629.491610:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207888e00 x1862022294215296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767629.491616:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207888e00 x1862022294215296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767629.491621:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767629.491623:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767629.491624:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767629.491625:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767629.491627:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207888e00 x1862022294215296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767629.491631:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767629.491634:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207888e00 x1862022294215296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767654 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767629.491638:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767629.491640:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767629.491641:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767629.491641:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767629.491642:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767629.491644:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767629.491646:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767629.491648:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767629.491649:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767629.491651:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207888e00 x1862022294215296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767654 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767629.491656:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207888e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294215296:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767629.491658:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767629.491660:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207889f80 x1862022294215424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767629.491664:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767629.491665:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767629.491667:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767629.491668:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767629.491670:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767629.491672:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207889f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294215424:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767629.491674:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767629.491675:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767629.491676:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767629.491677:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767629.491679:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b1000. 02000000:00000001:3.0:1775767629.491680:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767629.491682:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767629.491684:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b9a0. 00000400:00000010:3.0:1775767629.491686:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e088. 00000100:00000200:3.0:1775767629.491688:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294215424, portal 4 00000100:00000001:3.0:1775767629.491690:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767629.491691:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606126464 : -112966103425152 : ffff994207889f80) 00000100:00000040:3.0:1775767629.491695:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207889f80 x1862022294215424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767654 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767629.491699:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767629.491700:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767629.491702:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e660. 00000100:00000200:3.0:1775767629.491703:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294215424, offset 0 00000400:00000200:3.0:1775767629.491706:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767629.491708:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767629.491710:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767629.491712:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767629.491714:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767629.491715:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767629.491717:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767629.491719:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224856:224856:256:4294967295] 192.168.202.156@tcp LPNI seq info [224852:224852:8:4294967295] 00000400:00000200:3.0:1775767629.491722:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767629.491726:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767629.491729:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767629.491731:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8200. 00000800:00000200:3.0:1775767629.491733:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767629.491735:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1775767629.491738:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767629.491740:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000100:3.0:1775767629.491742:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767629.491744:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8200. 00000400:00000200:3.0:1775767629.491746:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767629.491748:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767629.491750:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e660 00000400:00000010:3.0:1775767629.491751:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e660. 00000100:00000001:3.0:1775767629.491752:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767629.491754:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207889f80 x1862022294215424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767654 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767629.491764:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767629.491766:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207889f80 x1862022294215424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767654 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767629.491770:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767629.491771:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767629.491773:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767629.491775:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:3.0:1775767629.491776:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767629.491778:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299554339 00000100:00000001:3.0:1775767629.491779:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299555645 00000100:00000001:3.0:1775767629.491781:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299554339 00000100:00000001:3.0:1775767629.491782:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299555645 00000100:00000001:3.0:1775767629.491784:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767629.491785:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767629.491786:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767629.491788:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767629/real 1775767629] req@ffff994207889f80 x1862022294215424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767629.491793:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767629.491793:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767629.491794:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b9a0. 00000400:00000200:3.0:1775767629.491796:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e088 00000400:00000010:3.0:1775767629.491797:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e088. 00000100:00000001:3.0:1775767629.491798:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767629.491800:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207889f80 x1862022294215424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767629.491806:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207889f80 x1862022294215424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767629.491809:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767629.491811:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767629.491813:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767629.491814:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767629.491816:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207889f80 x1862022294215424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767629.491820:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767629.491822:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207889f80 x1862022294215424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767654 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767629.491826:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767629.491827:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767629.491828:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767629.491829:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767629.491830:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767629.491832:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767629.491834:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767629.491836:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767629.491837:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767629.491839:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207889f80 x1862022294215424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767654 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767629.491844:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207889f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294215424:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767629.491846:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767629.491847:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767629.491849:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff99420788b100 x1862022294215168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767654 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767629.491854:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767629.491855:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767629.491856:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b2800. 02000000:00000001:3.0:1775767629.491858:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767629.491859:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767629.491860:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:3.0:1775767629.491862:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767629.491863:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543fc00. 02000000:00000001:3.0:1775767629.491865:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767629.491865:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767629.491867:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff99420788b100. 00000100:00000001:3.0:1775767629.491868:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767629.491869:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767629.491870:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767629.491872:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207888e00 x1862022294215296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767654 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767629.491876:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767629.491877:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767629.491878:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b2c00. 02000000:00000001:3.0:1775767629.491879:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767629.491880:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767629.491881:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:3.0:1775767629.491882:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767629.491883:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543c400. 02000000:00000001:3.0:1775767629.491884:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767629.491884:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767629.491885:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207888e00. 00000100:00000001:3.0:1775767629.491887:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767629.491887:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767629.491888:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767629.491890:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207889f80 x1862022294215424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767654 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767629.491893:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767629.491894:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767629.491895:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b1000. 02000000:00000001:3.0:1775767629.491896:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767629.491897:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767629.491897:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:3.0:1775767629.491898:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767629.491899:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543e800. 02000000:00000001:3.0:1775767629.491901:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767629.491903:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767629.491904:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207889f80. 00000100:00000001:3.0:1775767629.491907:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767629.491908:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767629.491910:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767629.491913:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767629.491915:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767629.491917:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767629.491918:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767629.491920:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767629.491923:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207888700 x1862022294215552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767629.491931:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767629.491932:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767629.491935:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767629.491937:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767629.491939:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767629.491942:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207888700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294215552:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767629.491946:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767629.491948:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767629.491949:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767629.491951:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767629.491955:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b1000. 02000000:00000001:3.0:1775767629.491957:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767629.491960:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767629.491963:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b9a0. 00000400:00000010:3.0:1775767629.491966:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e088. 00000100:00000200:3.0:1775767629.491969:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294215552, portal 4 00000100:00000001:3.0:1775767629.491971:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767629.491972:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606120192 : -112966103431424 : ffff994207888700) 00000100:00000040:3.0:1775767629.491978:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207888700 x1862022294215552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767654 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767629.491983:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767629.491984:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767629.491986:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e660. 00000100:00000200:3.0:1775767629.491988:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294215552, offset 0 00000400:00000200:3.0:1775767629.491991:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767629.491994:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767629.491997:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767629.491999:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767629.492001:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767629.492003:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767629.492004:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767629.492007:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224857:224857:256:4294967295] 192.168.202.156@tcp LPNI seq info [224853:224853:8:4294967295] 00000400:00000200:3.0:1775767629.492010:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767629.492014:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767629.492017:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767629.492019:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8200. 00000800:00000200:3.0:1775767629.492021:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767629.492023:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1775767629.492026:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767629.492028:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000100:3.0:1775767629.492030:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767629.492032:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8200. 00000400:00000200:3.0:1775767629.492034:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767629.492036:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767629.492038:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e660 00000400:00000010:3.0:1775767629.492039:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e660. 00000100:00000001:3.0:1775767629.492041:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767629.492043:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207888700 x1862022294215552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767654 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767629.492049:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767629.492051:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207888700 x1862022294215552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767654 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767629.492055:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767629.492056:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767629.492058:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767629.492060:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:3.0:1775767629.492062:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767629.492063:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299554342 00000100:00000001:3.0:1775767629.492065:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299555645 00000100:00000001:3.0:1775767629.492067:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299554342 00000100:00000001:3.0:1775767629.492068:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299555645 00000100:00000001:3.0:1775767629.492070:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767629.492071:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767629.492072:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767629.492074:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767629/real 1775767629] req@ffff994207888700 x1862022294215552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767629.492078:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767629.492079:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767629.492081:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b9a0. 00000400:00000200:3.0:1775767629.492082:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e088 00000400:00000010:3.0:1775767629.492083:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e088. 00000100:00000001:3.0:1775767629.492085:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767629.492087:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207888700 x1862022294215552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767629.492091:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207888700 x1862022294215552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767629.492095:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767629.492096:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767629.492097:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767629.492098:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767629.492100:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207888700 x1862022294215552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767629.492104:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767629.492106:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207888700 x1862022294215552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767654 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767629.492112:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767629.492113:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767629.492115:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767629.492116:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767629.492117:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767629.492119:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767629.492120:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767629.492122:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767629.492123:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767629.492126:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207888700 x1862022294215552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767654 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767629.492130:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207888700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294215552:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767629.492133:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767629.492134:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767629.492136:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207888700 x1862022294215552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767654 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767629.492140:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767629.492141:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767629.492142:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b1000. 02000000:00000001:3.0:1775767629.492143:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767629.492144:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767629.492145:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:3.0:1775767629.492147:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767629.492148:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543f800. 02000000:00000001:3.0:1775767629.492149:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767629.492150:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767629.492152:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207888700. 00000100:00000001:3.0:1775767629.492168:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767629.492169:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767629.492171:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767629.492173:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767629.492174:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767629.492176:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767629.492177:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767629.553349:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775767629.553352:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775767629.553355:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775767630.513269:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767630.513277:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767630.513280:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767630.513287:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767630.513294:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767630.513301:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767630.513306:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767630.513309:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767630.513312:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767631.537238:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767631.537246:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767631.537248:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767631.537258:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767631.537267:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767631.537273:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767631.537277:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767631.537281:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767631.537302:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767631.601300:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775767631.601303:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775767631.601305:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775767632.561231:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767632.561236:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767632.561239:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767632.561244:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767632.561248:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767632.561252:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767632.561263:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767632.561266:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767632.561271:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767633.585269:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767633.585275:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767633.585277:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767633.585284:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767633.585290:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767633.585296:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767633.585304:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767633.585309:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767633.585318:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767633.649347:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775767633.649352:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775767633.649356:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775767634.609309:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767634.609314:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767634.609316:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767634.609320:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767634.609326:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767634.609330:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767634.609336:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767634.609339:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767634.609343:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767634.609614:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767634.609618:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767634.609621:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767634.609630:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207888700 x1862022294215680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767634.609644:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767634.609646:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767634.609650:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767634.609654:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767634.609657:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767634.609661:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207888700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294215680:192.168.202.156@tcp:250:kworker.0 00000100:00000001:3.0:1775767634.609668:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767634.609685:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767634.609688:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767634.609692:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767634.609699:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b1000. 02000000:00000001:3.0:1775767634.609703:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767634.609712:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767634.609718:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b9a0. 00000400:00000010:3.0:1775767634.609725:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e088. 00000100:00000200:3.0:1775767634.609734:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294215680, portal 25 00000100:00000001:3.0:1775767634.609739:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767634.609740:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606120192 : -112966103431424 : ffff994207888700) 00000100:00000040:3.0:1775767634.609746:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207888700 x1862022294215680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767659 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767634.609760:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767634.609763:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767634.609768:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e660. 00000100:00000200:3.0:1775767634.609774:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294215680, offset 0 00000400:00000200:3.0:1775767634.609783:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767634.609793:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767634.609803:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767634.609812:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767634.609820:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767634.609827:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767634.609832:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767634.609841:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224858:224858:256:4294967295] 192.168.202.156@tcp LPNI seq info [224854:224854:8:4294967295] 00000400:00000200:3.0:1775767634.609855:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767634.609876:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767634.609892:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767634.609901:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8200. 00000800:00000200:3.0:1775767634.609915:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767634.609928:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:3.0:1775767634.609947:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767634.609956:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767634.609960:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299560765 00000100:00000010:3.0:1775767634.609967:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207981d80. 00000100:00000001:3.0:1775767634.609976:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1775767634.609983:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299560765 00000100:00000010:3.0:1775767634.609989:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207981240. 00000100:00000001:3.0:1775767634.609993:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1775767634.609998:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767634.610003:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767634.610006:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767634.610010:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767634.610017:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767634.610020:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1775767634.610035:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767634.610038:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767634.610041:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767634.610048:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207889f80 x1862022294215808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767634.610065:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767634.610067:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767634.610071:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767634.610075:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767634.610080:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767634.610085:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207889f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294215808:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767634.610093:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767634.610096:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767634.610099:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767634.610103:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767634.610108:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b2c00. 02000000:00000001:3.0:1775767634.610111:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767634.610119:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767634.610124:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b2c0. 00000400:00000010:3.0:1775767634.610130:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e440. 00000100:00000200:3.0:1775767634.610135:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294215808, portal 10 00000100:00000001:3.0:1775767634.610139:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767634.610141:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606126464 : -112966103425152 : ffff994207889f80) 00000100:00000040:3.0:1775767634.610151:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207889f80 x1862022294215808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767659 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767634.610191:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767634.610196:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767634.610200:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ef68. 00000100:00000200:3.0:1775767634.610204:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294215808, offset 0 00000400:00000200:3.0:1775767634.610212:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767634.610219:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767634.610227:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767634.610231:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767634.610238:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767634.610242:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767634.610246:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767634.610252:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224859:224859:255:4294967295] 192.168.202.156@tcp LPNI seq info [224855:224855:7:4294967295] 00000400:00000200:3.0:1775767634.610262:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767634.610282:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767634.610301:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767634.610311:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8000. 00000800:00000200:3.0:1775767634.610321:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767634.610334:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:3.0:1775767634.610351:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767634.610358:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:3.0:1775767634.610368:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767634.610372:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299559443 00000100:00000001:3.0:1775767634.610378:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299560765 00000100:00000001:3.0:1775767634.610383:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299559443 00000100:00000001:3.0:1775767634.610388:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299560765 00000100:00000001:3.0:1775767634.610392:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767634.610397:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767634.610402:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767634.610407:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767634.610414:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767634.610419:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1775767634.610424:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767634.610444:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767634.610448:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767634.610456:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207888e00 x1862022294215936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767634.610473:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767634.610476:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767634.610480:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767634.610483:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767634.610495:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767634.610502:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207888e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294215936:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767634.610508:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767634.610511:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767634.610514:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767634.610518:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767634.610534:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b2800. 02000000:00000001:3.0:1775767634.610541:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767634.610547:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767634.610552:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b210. 00000400:00000010:3.0:1775767634.610558:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e110. 00000100:00000200:3.0:1775767634.610564:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294215936, portal 10 00000100:00000001:3.0:1775767634.610568:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767634.610572:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606121984 : -112966103429632 : ffff994207888e00) 00000100:00000040:3.0:1775767634.610586:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207888e00 x1862022294215936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767659 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767634.610605:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767634.610610:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767634.610616:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e908. 00000100:00000200:3.0:1775767634.610620:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294215936, offset 0 00000400:00000200:3.0:1775767634.610625:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767634.610631:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767634.610635:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767634.610639:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767634.610642:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767634.610644:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767634.610646:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767634.610649:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224860:224860:254:4294967295] 192.168.202.156@tcp LPNI seq info [224856:224856:6:4294967295] 00000400:00000200:3.0:1775767634.610653:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767634.610659:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767634.610669:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767634.610673:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8300. 00000800:00000200:3.0:1775767634.610677:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767634.610681:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:3.0:1775767634.610684:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767634.610687:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:3.0:1775767634.610690:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767634.610692:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299559435 00000100:00000001:3.0:1775767634.610694:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299560765 00000100:00000001:3.0:1775767634.610696:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299559435 00000100:00000001:3.0:1775767634.610698:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299560765 00000100:00000001:3.0:1775767634.610700:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767634.610702:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767634.610704:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767634.610707:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff99420788b100 x1862022294216064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767634.610714:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767634.610716:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767634.610718:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767634.610720:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767634.610722:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767634.610725:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff99420788b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294216064:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767634.610727:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767634.610730:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767634.610731:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767634.610733:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767634.610737:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b0000. 02000000:00000001:3.0:1775767634.610739:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767634.610742:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767634.610744:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780bb00. 00000400:00000010:3.0:1775767634.610747:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e198. 00000100:00000200:3.0:1775767634.610756:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294216064, portal 4 00000100:00000001:3.0:1775767634.610758:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767634.610759:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606130944 : -112966103420672 : ffff99420788b100) 00000100:00000040:3.0:1775767634.610764:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff99420788b100 x1862022294216064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767659 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767634.610771:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767634.610773:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767634.610776:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767634.610778:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294216064, offset 0 00000400:00000200:3.0:1775767634.610782:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767634.610786:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767634.610789:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767634.610792:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767634.610795:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767634.610797:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767634.610799:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767634.610802:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224861:224861:253:4294967295] 192.168.202.156@tcp LPNI seq info [224857:224857:5:4294967295] 00000400:00000200:3.0:1775767634.610806:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767634.610812:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767634.610817:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767634.610821:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8700. 00000800:00000200:3.0:1775767634.610825:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767634.610829:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:3.0:1775767634.610832:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767634.610835:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:3.0:1775767634.610838:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767634.610840:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299559458 00000100:00000001:3.0:1775767634.610842:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299560765 00000100:00000001:3.0:1775767634.610844:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299559458 00000100:00000001:3.0:1775767634.610846:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299560765 00000100:00000001:3.0:1775767634.610848:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767634.610851:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767634.610853:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767634.610855:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767634.610859:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767634.610861:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1775767634.610862:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767634.610864:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767634.610866:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767634.610870:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207889880 x1862022294216192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767634.610876:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767634.610878:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767634.610881:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767634.610883:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767634.610885:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767634.610888:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207889880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294216192:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767634.610891:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767634.610893:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767634.610895:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767634.610897:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767634.610900:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b0c00. 02000000:00000001:3.0:1775767634.610902:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767634.610906:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767634.610909:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b840. 00000400:00000010:3.0:1775767634.610912:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e770. 00000100:00000200:3.0:1775767634.610915:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294216192, portal 4 00000100:00000001:3.0:1775767634.610918:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767634.610919:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606124672 : -112966103426944 : ffff994207889880) 00000100:00000040:3.0:1775767634.610925:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207889880 x1862022294216192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767659 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767634.610933:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767634.610935:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767634.610939:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767634.610942:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294216192, offset 0 00000400:00000200:3.0:1775767634.610946:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767634.610951:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767634.611048:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767634.611052:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767634.611054:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767634.611057:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767634.611059:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767634.611063:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224862:224862:252:4294967295] 192.168.202.156@tcp LPNI seq info [224858:224858:4:4294967295] 00000400:00000200:3.0:1775767634.611068:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767634.611075:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767634.611080:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767634.611084:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767634.611088:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767634.611093:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:3.0:1775767634.611096:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767634.611100:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:3.0:1775767634.611103:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767634.611105:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299559461 00000100:00000001:3.0:1775767634.611108:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299560765 00000100:00000001:3.0:1775767634.611111:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299559461 00000100:00000001:3.0:1775767634.611113:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299560765 00000100:00000001:3.0:1775767634.611115:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767634.611118:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767634.611121:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767634.611124:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767634.611128:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767634.611130:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1775767634.611131:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767634.611133:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767634.611135:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767634.611136:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767635.633284:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767635.633293:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767635.633297:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767635.633306:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767635.633313:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767635.633319:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767635.633322:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767635.633326:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767635.633329:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767635.697411:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775767635.697429:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775767635.697433:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775767636.657233:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767636.657241:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767636.657243:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767636.657248:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767636.657253:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767636.657257:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767636.657263:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767636.657265:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767636.657267:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767637.681264:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767637.681272:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767637.681274:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767637.681279:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767637.681283:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767637.681288:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767637.681295:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767637.681300:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767637.681307:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:3.0:1775767637.681524:0:2389:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.156:988 00000400:00000100:3.0:1775767637.681572:0:2389:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.156@tcp at host 192.168.202.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:3.0:1775767637.681583:0:2389:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767637.681586:0:2389:0:(router.c:1750:lnet_notify()) 192.168.202.56@tcp notifying 192.168.202.156@tcp: down 00000400:00000200:3.0:1775767637.681591:0:2389:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767637.681595:0:2389:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:3.0:1775767637.681601:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000800:00000010:3.0:1775767637.681605:0:2389:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8200. 00000400:00000200:3.0:1775767637.681609:0:2389:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1775767637.681614:0:2389:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1775767637.681619:0:2389:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e660 00000400:00000010:3.0:1775767637.681622:0:2389:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e660. 00000100:00000001:3.0:1775767637.681626:0:2389:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767637.681636:0:2389:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff994207888700 x1862022294215680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767659 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767637.681671:0:2389:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767637.681677:0:2389:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207888700 x1862022294215680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767659 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767637.681686:0:2389:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767637.681688:0:2389:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1775767637.681706:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000800:00000010:3.0:1775767637.681709:0:2389:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8000. 00000400:00000200:3.0:1775767637.681712:0:2389:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1775767637.681715:0:2389:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1775767637.681719:0:2389:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ef68 00000400:00000010:3.0:1775767637.681721:0:2389:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ef68. 00000100:00000001:3.0:1775767637.681724:0:2389:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767637.681729:0:2389:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff994207889f80 x1862022294215808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767659 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767637.681738:0:2389:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767637.681742:0:2389:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207889f80 x1862022294215808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767659 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767637.681750:0:2389:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767637.681751:0:2389:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1775767637.681757:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000800:00000010:3.0:1775767637.681760:0:2389:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8300. 00000400:00000200:3.0:1775767637.681763:0:2389:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1775767637.681783:0:2389:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1775767637.681787:0:2389:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e908 00000400:00000010:3.0:1775767637.681789:0:2389:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e908. 00000100:00000001:3.0:1775767637.681793:0:2389:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767637.681800:0:2389:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff994207888e00 x1862022294215936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767659 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767637.681810:0:2389:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767637.681813:0:2389:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207888e00 x1862022294215936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767659 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767637.681821:0:2389:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767637.681823:0:2389:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1775767637.681829:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000800:00000010:3.0:1775767637.681831:0:2389:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8700. 00000400:00000200:3.0:1775767637.681835:0:2389:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1775767637.681838:0:2389:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1775767637.681841:0:2389:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767637.681844:0:2389:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767637.681863:0:2389:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767637.681872:0:2389:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff99420788b100 x1862022294216064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767659 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767637.681882:0:2389:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767637.681885:0:2389:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff99420788b100 x1862022294216064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767659 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767637.681891:0:2389:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767637.681894:0:2389:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1775767637.681899:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000800:00000010:3.0:1775767637.681902:0:2389:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767637.681906:0:2389:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1775767637.681910:0:2389:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1775767637.681913:0:2389:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767637.681915:0:2389:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767637.681923:0:2389:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767637.681929:0:2389:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff994207889880 x1862022294216192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767659 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767637.681938:0:2389:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767637.681941:0:2389:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207889880 x1862022294216192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767659 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767637.681948:0:2389:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767637.681950:0:2389:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:3.0:1775767637.745467:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775767637.745471:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775767637.745475:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775767638.705267:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767638.705273:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767638.705275:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767638.705281:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767638.705285:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767638.705290:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767638.705295:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767638.705300:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767638.705310:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767639.729262:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767639.729268:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767639.729270:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767639.729274:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767639.729282:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767639.729286:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767639.729290:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767639.729293:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767639.729303:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775767639.730256:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767639.730264:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775767639.731440:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767639.731444:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775767639.731783:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767639.731785:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775767639.732111:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767639.732114:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775767639.732637:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767639.732640:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000020:00000001:3.0:1775767639.793408:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775767639.793415:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775767639.793421:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775767640.753314:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767640.753321:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767640.753324:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767640.753332:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767640.753344:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767640.753352:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767640.753361:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767640.753365:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767640.753381:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767640.753434:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767640.753438:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767640.753440:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767640.753444:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767640.753447:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767640.753451:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767640.753453:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767640.753455:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767640.753458:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767641.777228:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767641.777234:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767641.777236:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767641.777241:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767641.777245:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767641.777249:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767641.777251:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767641.777253:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767641.777256:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767641.841395:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775767641.841402:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775767641.841407:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775767642.801263:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767642.801271:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767642.801274:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767642.801283:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767642.801291:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767642.801297:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767642.801311:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767642.801314:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767642.801318:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767643.825228:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767643.825236:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767643.825240:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767643.825244:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767643.825250:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767643.825255:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767643.825267:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767643.825269:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767643.825271:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767643.889280:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775767643.889284:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775767643.889287:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775767644.849232:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767644.849238:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767644.849243:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767644.849250:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767644.849258:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767644.849263:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767644.849276:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767644.849280:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767644.849284:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775767644.849810:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767644.849815:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775767644.850466:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767644.850472:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775767644.850715:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767644.850717:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775767644.850947:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767644.850949:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775767644.851124:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767644.851126:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000100:00000001:3.0:1775767645.873224:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767645.873229:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767645.873233:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767645.873240:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767645.873246:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767645.873250:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767645.873254:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767645.873257:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767645.873260:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767645.937301:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775767645.937304:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775767645.937307:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775767646.897243:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767646.897250:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767646.897254:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767646.897269:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767646.897276:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767646.897281:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767646.897292:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767646.897295:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767646.897297:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767647.921221:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767647.921226:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767647.921228:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767647.921241:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767647.921245:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767647.921249:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767647.921264:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767647.921268:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767647.921271:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767647.985341:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775767647.985345:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775767647.985349:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775767648.945235:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767648.945242:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767648.945247:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767648.945255:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767648.945261:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767648.945267:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767648.945279:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767648.945282:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767648.945286:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767649.969274:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767649.969304:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767649.969306:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767649.969310:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767649.969316:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767649.969321:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767649.969328:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767649.969333:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767649.969337:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767649.969592:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767649.969598:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767649.969602:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767649.969614:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfce00 x1862022294217600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767649.969641:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767649.969646:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767649.969652:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767649.969656:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767649.969661:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767649.969671:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294217600:192.168.202.156@tcp:250:kworker.0 00000100:00000001:3.0:1775767649.969681:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767649.969686:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767649.969689:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767649.969692:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767649.969699:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b0400. 02000000:00000001:3.0:1775767649.969703:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767649.969722:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767649.969730:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767649.969739:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767649.969748:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294217600, portal 25 00000100:00000001:3.0:1775767649.969755:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767649.969758:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611840000 : -112966097711616 : ffff994207dfce00) 00000100:00000040:3.0:1775767649.969771:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfce00 x1862022294217600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767674 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767649.969795:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767649.969802:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767649.969806:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767649.969812:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294217600, offset 0 00000400:00000200:3.0:1775767649.969824:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767649.969834:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767649.969842:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767649.969851:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767649.969858:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767649.969864:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767649.969870:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767649.969878:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224873:224873:256:4294967295] 192.168.202.156@tcp LPNI seq info [224869:224869:8:4294967295] 00000400:00000200:3.0:1775767649.969891:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767649.969910:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767649.969918:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767649.969926:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767649.969938:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767649.969944:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775767649.969955:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767649.969961:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000100:3.0:1775767649.969974:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767649.969978:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767649.969987:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767649.970000:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767649.970010:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767649.970015:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767649.970021:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767649.970029:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfce00 x1862022294217600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767674 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767649.970044:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767649.970051:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfce00 x1862022294217600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767674 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767649.970070:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767649.970072:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767649.970079:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767649.970086:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767649.970088:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299576125 00000100:00000010:3.0:1775767649.970095:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207981600. 00000100:00000001:3.0:1775767649.970104:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1775767649.970112:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299576125 00000100:00000010:3.0:1775767649.970116:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207981a80. 00000100:00000001:3.0:1775767649.970121:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1775767649.970127:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767649.970131:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767649.970133:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767649.970140:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767649/real 1775767649] req@ffff994207dfce00 x1862022294217600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767649.970185:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767649.970187:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767649.970190:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767649.970196:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767649.970200:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767649.970206:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767649.970216:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfce00 x1862022294217600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767649.970241:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfce00 x1862022294217600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767649.970248:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767649.970250:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767649.970253:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767649.970255:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767649.970258:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfce00 x1862022294217600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767649.970266:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767649.970270:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfce00 x1862022294217600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767674 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767649.970293:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767649.970297:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767649.970299:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767649.970301:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767649.970303:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767649.970306:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767649.970309:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767649.970313:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767649.970315:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767649.970321:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfce00 x1862022294217600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767674 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767649.970340:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294217600:192.168.202.156@tcp:250:kworker.0 00000100:00000001:3.0:1775767649.970349:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767649.970355:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767649.970360:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfce00 x1862022294217600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767674 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767649.970377:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767649.970381:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767649.970386:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b0400. 02000000:00000001:3.0:1775767649.970394:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767649.970399:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767649.970403:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:3.0:1775767649.970408:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767649.970412:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1e400. 02000000:00000001:3.0:1775767649.970419:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767649.970422:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767649.970425:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfce00. 00000100:00000001:3.0:1775767649.970431:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767649.970434:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767649.970439:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767649.970447:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767649.970449:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767649.970454:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767649.970455:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767649.970458:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767649.970465:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dff480 x1862022294217728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767649.970477:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767649.970479:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767649.970484:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767649.970486:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767649.970489:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767649.970492:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294217728:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767649.970497:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767649.970500:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767649.970501:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767649.970503:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767649.970507:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b0400. 02000000:00000001:3.0:1775767649.970508:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767649.970513:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767649.970516:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767649.970521:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767649.970524:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294217728, portal 10 00000100:00000001:3.0:1775767649.970526:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767649.970528:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611849856 : -112966097701760 : ffff994207dff480) 00000100:00000040:3.0:1775767649.970535:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dff480 x1862022294217728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767674 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767649.970542:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767649.970544:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767649.970546:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767649.970548:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294217728, offset 0 00000400:00000200:3.0:1775767649.970551:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767649.970554:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767649.970557:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767649.970559:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767649.970561:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767649.970563:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767649.970565:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767649.970567:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224874:224874:256:4294967295] 192.168.202.156@tcp LPNI seq info [224870:224870:8:4294967295] 00000400:00000200:3.0:1775767649.970570:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767649.970575:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767649.970578:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767649.970580:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767649.970584:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767649.970586:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775767649.970589:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767649.970591:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000100:3.0:1775767649.970594:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767649.970595:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767649.970597:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767649.970600:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767649.970602:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767649.970603:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767649.970606:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767649.970609:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dff480 x1862022294217728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767674 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767649.970617:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767649.970620:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dff480 x1862022294217728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767674 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767649.970627:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767649.970628:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767649.970632:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767649.970636:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:3.0:1775767649.970640:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767649.970642:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299574803 00000100:00000001:3.0:1775767649.970645:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299576125 00000100:00000001:3.0:1775767649.970647:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299574803 00000100:00000001:3.0:1775767649.970649:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299576125 00000100:00000001:3.0:1775767649.970652:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767649.970654:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767649.970656:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767649.970660:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767649/real 1775767649] req@ffff994207dff480 x1862022294217728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767649.970667:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767649.970667:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767649.970669:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767649.970671:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767649.970672:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767649.970673:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767649.970675:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dff480 x1862022294217728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767649.970681:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dff480 x1862022294217728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767649.970685:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767649.970686:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767649.970687:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767649.970688:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767649.970690:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dff480 x1862022294217728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767649.970694:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767649.970697:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dff480 x1862022294217728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767674 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767649.970701:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767649.970703:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767649.970704:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767649.970705:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767649.970706:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767649.970708:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767649.970709:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767649.970712:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767649.970713:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767649.970716:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dff480 x1862022294217728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767674 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767649.970722:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294217728:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767649.970726:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767649.970729:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfed80 x1862022294217856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767649.970736:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767649.970737:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767649.970739:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767649.970741:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767649.970743:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767649.970746:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294217856:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767649.970750:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767649.970752:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767649.970753:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767649.970755:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767649.970759:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b2000. 02000000:00000001:3.0:1775767649.970761:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767649.970765:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767649.970768:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767649.970771:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767649.970774:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294217856, portal 10 00000100:00000001:3.0:1775767649.970776:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767649.970777:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611848064 : -112966097703552 : ffff994207dfed80) 00000100:00000040:3.0:1775767649.970782:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfed80 x1862022294217856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767674 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767649.970786:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767649.970788:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767649.970789:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767649.970791:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294217856, offset 0 00000400:00000200:3.0:1775767649.970794:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767649.970797:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767649.970800:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767649.970802:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767649.970804:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767649.970805:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767649.970807:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767649.970809:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224875:224875:256:4294967295] 192.168.202.156@tcp LPNI seq info [224871:224871:8:4294967295] 00000400:00000200:3.0:1775767649.970812:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767649.970816:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767649.970819:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767649.970821:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767649.970824:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767649.970826:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775767649.970828:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767649.970830:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000100:3.0:1775767649.970833:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767649.970834:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767649.970836:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767649.970839:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767649.970841:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767649.970842:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767649.970843:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767649.970845:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfed80 x1862022294217856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767674 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767649.970851:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767649.970852:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfed80 x1862022294217856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767674 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767649.970857:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767649.970858:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767649.970860:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767649.970862:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:3.0:1775767649.970864:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767649.970865:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299574795 00000100:00000001:3.0:1775767649.970867:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299576125 00000100:00000001:3.0:1775767649.970869:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299574795 00000100:00000001:3.0:1775767649.970870:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299576125 00000100:00000001:3.0:1775767649.970871:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767649.970872:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767649.970874:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767649.970876:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767649/real 1775767649] req@ffff994207dfed80 x1862022294217856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767649.970880:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767649.970881:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767649.970882:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767649.970883:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767649.970884:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767649.970886:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767649.970888:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfed80 x1862022294217856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767649.970892:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfed80 x1862022294217856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767649.970898:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767649.970899:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767649.970900:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767649.970902:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767649.970905:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfed80 x1862022294217856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767649.970912:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767649.970916:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfed80 x1862022294217856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767674 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767649.970923:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767649.970926:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767649.970927:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767649.970929:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767649.970930:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767649.970933:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767649.970936:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767649.970940:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767649.970942:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767649.970946:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfed80 x1862022294217856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767674 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767649.970955:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294217856:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767649.970957:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767649.970960:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfe300 x1862022294217984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767649.970966:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767649.970968:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767649.970970:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767649.970972:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767649.970974:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767649.970976:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294217984:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767649.970980:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767649.970981:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767649.970983:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767649.970985:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767649.970989:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b3400. 02000000:00000001:3.0:1775767649.970991:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767649.970996:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767649.970998:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767649.971002:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767649.971005:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294217984, portal 4 00000100:00000001:3.0:1775767649.971007:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767649.971008:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611845376 : -112966097706240 : ffff994207dfe300) 00000100:00000040:3.0:1775767649.971014:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfe300 x1862022294217984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767674 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767649.971021:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767649.971024:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767649.971026:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767649.971029:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294217984, offset 0 00000400:00000200:3.0:1775767649.971033:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767649.971037:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767649.971041:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767649.971044:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767649.971047:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767649.971050:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767649.971052:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767649.971054:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224876:224876:256:4294967295] 192.168.202.156@tcp LPNI seq info [224872:224872:8:4294967295] 00000400:00000200:3.0:1775767649.971057:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767649.971062:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767649.971065:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767649.971067:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767649.971070:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767649.971072:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775767649.971074:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767649.971076:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000100:3.0:1775767649.971079:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767649.971080:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767649.971082:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767649.971084:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767649.971086:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767649.971088:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767649.971089:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767649.971092:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfe300 x1862022294217984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767674 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767649.971097:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767649.971099:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfe300 x1862022294217984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767674 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767649.971103:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767649.971104:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767649.971106:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767649.971109:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:3.0:1775767649.971111:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767649.971112:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299574818 00000100:00000001:3.0:1775767649.971114:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299576125 00000100:00000001:3.0:1775767649.971115:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299574818 00000100:00000001:3.0:1775767649.971117:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299576125 00000100:00000001:3.0:1775767649.971118:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767649.971119:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767649.971121:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767649.971123:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767649/real 1775767649] req@ffff994207dfe300 x1862022294217984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767649.971128:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767649.971128:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767649.971130:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767649.971131:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767649.971132:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767649.971133:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767649.971135:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfe300 x1862022294217984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767649.971141:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfe300 x1862022294217984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767649.971145:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767649.971146:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767649.971148:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767649.971149:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767649.971151:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfe300 x1862022294217984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767649.971171:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767649.971175:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfe300 x1862022294217984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767674 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767649.971182:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767649.971184:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767649.971185:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767649.971187:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767649.971189:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767649.971192:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767649.971195:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767649.971198:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767649.971201:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767649.971205:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfe300 x1862022294217984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767674 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767649.971214:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294217984:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767649.971218:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767649.971220:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767649.971223:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dff480 x1862022294217728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767674 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767649.971230:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767649.971232:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767649.971234:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b0400. 02000000:00000001:3.0:1775767649.971237:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767649.971238:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767649.971240:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:3.0:1775767649.971242:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767649.971244:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1fc00. 02000000:00000001:3.0:1775767649.971247:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767649.971248:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767649.971249:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dff480. 00000100:00000001:3.0:1775767649.971252:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767649.971253:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767649.971255:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767649.971259:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfed80 x1862022294217856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767674 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767649.971266:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767649.971267:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767649.971269:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b2000. 02000000:00000001:3.0:1775767649.971271:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767649.971272:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767649.971274:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:3.0:1775767649.971276:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767649.971277:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1f800. 02000000:00000001:3.0:1775767649.971291:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767649.971292:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767649.971293:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfed80. 00000100:00000001:3.0:1775767649.971296:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767649.971297:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767649.971299:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767649.971302:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfe300 x1862022294217984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767674 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767649.971306:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767649.971307:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767649.971308:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b3400. 02000000:00000001:3.0:1775767649.971310:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767649.971311:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767649.971312:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:3.0:1775767649.971313:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767649.971314:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1f000. 02000000:00000001:3.0:1775767649.971315:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767649.971316:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767649.971317:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfe300. 00000100:00000001:3.0:1775767649.971318:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767649.971319:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767649.971320:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767649.971323:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767649.971324:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767649.971325:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767649.971326:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767649.971327:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767649.971329:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfc700 x1862022294218112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767649.971334:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767649.971335:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767649.971336:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767649.971337:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767649.971339:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767649.971341:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294218112:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767649.971342:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767649.971344:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767649.971345:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767649.971346:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767649.971348:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b3400. 02000000:00000001:3.0:1775767649.971349:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767649.971351:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767649.971353:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767649.971355:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767649.971357:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294218112, portal 4 00000100:00000001:3.0:1775767649.971358:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767649.971359:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611838208 : -112966097713408 : ffff994207dfc700) 00000100:00000040:3.0:1775767649.971363:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfc700 x1862022294218112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767674 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767649.971369:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767649.971372:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767649.971374:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767649.971377:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294218112, offset 0 00000400:00000200:3.0:1775767649.971381:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767649.971385:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767649.971388:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767649.971391:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767649.971394:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767649.971396:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767649.971398:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767649.971402:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224877:224877:256:4294967295] 192.168.202.156@tcp LPNI seq info [224873:224873:8:4294967295] 00000400:00000200:3.0:1775767649.971407:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767649.971413:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767649.971418:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767649.971421:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767649.971426:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767649.971429:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775767649.971433:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767649.971435:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000100:3.0:1775767649.971439:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767649.971441:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767649.971444:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767649.971446:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767649.971448:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767649.971450:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767649.971451:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767649.971454:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfc700 x1862022294218112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767674 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767649.971470:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767649.971474:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfc700 x1862022294218112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767674 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767649.971483:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767649.971485:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767649.971488:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767649.971491:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:3.0:1775767649.971495:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767649.971497:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299574822 00000100:00000001:3.0:1775767649.971499:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299576125 00000100:00000001:3.0:1775767649.971501:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299574822 00000100:00000001:3.0:1775767649.971503:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299576125 00000100:00000001:3.0:1775767649.971506:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767649.971508:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767649.971510:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767649.971514:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767649/real 1775767649] req@ffff994207dfc700 x1862022294218112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767649.971522:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767649.971524:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767649.971526:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767649.971528:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767649.971530:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767649.971532:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767649.971535:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfc700 x1862022294218112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767649.971544:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfc700 x1862022294218112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767649.971548:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767649.971550:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767649.971551:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767649.971552:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767649.971554:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfc700 x1862022294218112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767649.971559:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767649.971561:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfc700 x1862022294218112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767674 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767649.971565:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767649.971567:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767649.971568:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767649.971568:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767649.971570:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767649.971572:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767649.971573:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767649.971575:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767649.971577:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767649.971579:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfc700 x1862022294218112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767674 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767649.971585:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294218112:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767649.971588:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767649.971590:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767649.971593:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfc700 x1862022294218112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767674 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767649.971600:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767649.971602:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767649.971604:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b3400. 02000000:00000001:3.0:1775767649.971606:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767649.971608:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767649.971609:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:3.0:1775767649.971611:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767649.971613:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1ec00. 02000000:00000001:3.0:1775767649.971616:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767649.971618:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767649.971619:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfc700. 00000100:00000001:3.0:1775767649.971621:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767649.971623:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767649.971624:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767649.971628:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767649.971629:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767649.971631:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767649.971633:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767650.033330:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775767650.033333:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775767650.033336:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775767650.993243:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767650.993251:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767650.993254:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767650.993260:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767650.993265:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767650.993282:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767650.993292:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767650.993295:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767650.993298:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767652.017230:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767652.017237:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767652.017242:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767652.017248:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767652.017252:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767652.017256:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767652.017277:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767652.017281:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767652.017285:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767652.081418:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775767652.081423:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775767652.081426:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775767653.041269:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767653.041274:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767653.041277:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767653.041283:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767653.041288:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767653.041292:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767653.041295:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767653.041299:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767653.041302:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767654.065229:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767654.065235:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767654.065251:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767654.065261:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767654.065269:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767654.065275:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767654.065291:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767654.065294:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767654.065297:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767654.129314:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775767654.129317:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775767654.129320:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775767655.089266:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767655.089272:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767655.089274:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767655.089280:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767655.089286:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767655.089292:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767655.089297:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767655.089304:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767655.089310:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767655.089346:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767655.089350:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775767655.089354:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767655.089358:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767655.089361:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767655.089369:0:13471:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 4907 00000400:00000200:3.0:1775767655.089378:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767655.089385:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767655.089388:0:13471:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767655.089392:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767655.089398:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767655.089401:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767655.089405:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767655.089409:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767655.089412:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:3.0:1775767655.089414:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767655.089417:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:3.0:1775767655.089422:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:3.0:1775767655.089424:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:3.0:1775767655.089431:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767655.089435:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767655.089437:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767655.089441:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767655.089445:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:3.0:1775767655.089451:0:13471:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767655.089454:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767655.089459:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767655.089462:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767655.089467:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767655.089471:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775767655.089474:0:13471:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767655.089488:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8e00. 00000020:00000040:3.0:1775767655.089493:0:13471:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:3.0:1775767655.089498:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767655.089499:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767655.089504:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767655.089507:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767655.089512:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767655.089521:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b3400. 00000100:00000001:3.0:1775767655.089528:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767655.089538:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8e00 x1862022294218240/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767655.089555:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767655.089559:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767655.089565:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000864a6866] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8e00 x1862022294218240/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767655.089589:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767655.089594:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767655.089599:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775767655.089603:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767655.089606:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767655.089609:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767655.089613:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4907 00000400:00000200:3.0:1775767655.089619:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767655.089623:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767655.089628:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767655.089636:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767655.089638:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767655.089644:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767655.089650:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767655.089657:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767655.089661:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:3.0:1775767655.089664:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767655.089667:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:3.0:1775767655.089673:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:3.0:1775767655.089679:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:3.0:1775767655.089689:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767655.089695:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767655.089701:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767655.089707:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767655.089712:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00080000:3.0:1775767655.089720:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767655.089729:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767655.089733:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767655.089740:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767655.089744:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767655.089749:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767655.089754:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775767655.089760:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767655.089763:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767655.089769:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ab480. 00000020:00000040:3.0:1775767655.089774:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:3.0:1775767655.089781:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767655.089784:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767655.089787:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767655.089791:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767655.089794:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767655.089801:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2000. 00000100:00000001:3.0:1775767655.089808:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767655.089817:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ab480 x1862022294218368/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767655.089837:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767655.089841:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767655.089849:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000067a0ba74] to pc [ptlrpcd_rcv+-1] req@ffff9942099ab480 x1862022294218368/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767655.089869:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767655.089874:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767655.089880:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775767655.089884:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767655.089886:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767655.089893:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767655.089908:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4907 00000400:00000200:3.0:1775767655.089921:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767655.089931:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767655.089937:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767655.089946:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767655.089954:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:2.1:1775767655.089959:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000040:3.0:1775767655.089962:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000800:00000200:2.1:1775767655.089965:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000100:00000001:3.0:1775767655.089967:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767655.089973:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767655.089977:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:3.0:1775767655.089980:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767655.089984:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:3.0:1775767655.089989:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:3.0:1775767655.089994:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:3.0:1775767655.090000:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767655.090002:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767655.090006:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767655.090010:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767655.090012:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:3.0:1775767655.090016:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767655.090020:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767655.090024:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767655.090029:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767655.090031:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767655.090034:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767655.090037:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775767655.090042:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767655.090047:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767655.090053:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a9180. 00000020:00000040:3.0:1775767655.090058:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:3.0:1775767655.090062:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767655.090064:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767655.090066:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767655.090068:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767655.090071:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767655.090076:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0400. 00000100:00000001:3.0:1775767655.090080:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767655.090086:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a9180 x1862022294218496/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767655.090093:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767655.090097:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767655.090101:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bdc2586] to pc [ptlrpcd_rcv+-1] req@ffff9942099a9180 x1862022294218496/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767655.090126:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767655.090130:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767655.090138:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775767655.090141:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767655.090144:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767655.090150:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767655.090184:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4907 00000400:00000200:3.0:1775767655.090196:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767655.090204:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767655.090209:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767655.090218:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767655.090225:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767655.090436:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767655.090440:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767655.090445:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767655.090448:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:3.0:1775767655.090450:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767655.090452:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:3.0:1775767655.090454:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:3.0:1775767655.090456:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000800:00000200:2.1:1775767655.090462:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767655.090464:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000100:00000040:3.0:1775767655.090508:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767655.090511:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767655.090515:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767655.090520:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767655.090524:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:3.0:1775767655.090530:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767655.090536:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767655.090539:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767655.090542:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767655.090548:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767655.090552:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767655.090556:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:3.0:1775767655.090559:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767655.090564:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767655.090570:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8700. 00000020:00000040:3.0:1775767655.090572:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:3.0:1775767655.090576:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767655.090578:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767655.090580:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767655.090583:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767655.090587:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767655.090591:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b3400. 00000100:00000001:3.0:1775767655.090596:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767655.090607:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8700 x1862022294218624/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767655.090637:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767655.090641:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767655.090648:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000faa25e3b] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8700 x1862022294218624/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767655.090677:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767655.090683:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767655.090691:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775767655.090696:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767655.090699:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767655.090705:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767655.090712:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4907 00000400:00000200:3.0:1775767655.090723:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767655.090732:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767655.090737:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767655.090742:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767655.090745:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767655.090751:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767655.090755:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767655.090759:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767655.090761:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:3.0:1775767655.090765:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767655.090767:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:3.0:1775767655.090771:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:3.0:1775767655.090774:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:3.0:1775767655.090779:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767655.090784:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767655.090787:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767655.090790:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767655.090795:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:3.0:1775767655.090799:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767655.090802:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767655.090805:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767655.090808:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767655.090812:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767655.090817:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767655.090819:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:3.0:1775767655.090822:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767655.090825:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767655.090829:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8380. 00000020:00000040:3.0:1775767655.090833:0:13471:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:3.0:1775767655.090836:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767655.090838:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767655.090842:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767655.090844:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767655.090848:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767655.090853:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b1000. 00000100:00000001:3.0:1775767655.090857:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767655.090866:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8380 x1862022294218752/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767655.090879:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767655.090883:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767655.090890:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055f5621a] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8380 x1862022294218752/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767655.090912:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767655.090919:0:13471:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (4917) 00000800:00000200:2.1:1775767655.090938:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767655.090940:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775767655.091279:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767655.091281:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775767655.091495:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767655.091497:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000100:00000001:3.0:1775767656.113307:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767656.113317:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767656.113321:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767656.113333:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767656.113345:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767656.113350:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767656.113359:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767656.113369:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767656.113380:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767656.177363:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775767656.177369:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775767656.177373:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775767657.137262:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767657.137267:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767657.137270:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767657.137277:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767657.137281:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767657.137285:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767657.137299:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767657.137302:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767657.137304:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767658.161265:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767658.161271:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767658.161273:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767658.161279:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767658.161283:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767658.161288:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767658.161292:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767658.161296:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767658.161298:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767658.225350:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775767658.225354:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775767658.225358:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775767659.185266:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767659.185274:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767659.185277:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767659.185282:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767659.185287:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767659.185290:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767659.185293:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767659.185295:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767659.185299:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767660.209266:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767660.209272:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767660.209275:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767660.209283:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767660.209289:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767660.209293:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767660.209296:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767660.209301:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767660.209308:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767660.209521:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767660.209526:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767660.209529:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767660.209538:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fe300 x1862022294218880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767660.209566:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767660.209569:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767660.209575:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767660.209581:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767660.209587:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767660.209597:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294218880:192.168.202.156@tcp:250:kworker.0 00000100:00000001:3.0:1775767660.209617:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767660.209624:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767660.209626:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767660.209630:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767660.209636:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b2c00. 02000000:00000001:3.0:1775767660.209639:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767660.209651:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767660.209658:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767660.209667:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767660.209676:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294218880, portal 25 00000100:00000001:3.0:1775767660.209682:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767660.209685:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641205504 : -112966068346112 : ffff9942099fe300) 00000100:00000040:3.0:1775767660.209696:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fe300 x1862022294218880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767685 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767660.209727:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767660.209733:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767660.209739:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767660.209745:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294218880, offset 0 00000400:00000200:3.0:1775767660.209759:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767660.209773:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767660.209784:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767660.209795:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767660.209806:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767660.209814:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767660.209822:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767660.209830:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224883:224883:256:4294967295] 192.168.202.156@tcp LPNI seq info [224879:224879:8:4294967295] 00000400:00000200:3.0:1775767660.209841:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767660.209862:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767660.209875:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767660.209883:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767660.209897:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767660.209905:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775767660.209918:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767660.209924:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000100:3.0:1775767660.209935:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767660.209940:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767660.209946:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767660.209957:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767660.209967:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767660.209972:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767660.209977:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767660.209983:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fe300 x1862022294218880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767685 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767660.210017:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767660.210024:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fe300 x1862022294218880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767685 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767660.210052:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767660.210057:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767660.210065:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767660.210076:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767660.210083:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299586365 00000100:00000010:3.0:1775767660.210091:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207981660. 00000100:00000001:3.0:1775767660.210103:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1775767660.210113:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299586365 00000100:00000010:3.0:1775767660.210120:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207981ae0. 00000100:00000001:3.0:1775767660.210127:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1775767660.210132:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767660.210139:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767660.210145:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767660.210152:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767660/real 1775767660] req@ffff9942099fe300 x1862022294218880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767660.210216:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767660.210223:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767660.210227:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767660.210232:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767660.210236:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767660.210242:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767660.210249:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fe300 x1862022294218880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767660.210272:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fe300 x1862022294218880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767660.210294:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767660.210298:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767660.210304:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767660.210307:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767660.210315:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fe300 x1862022294218880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767660.210332:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767660.210343:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fe300 x1862022294218880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767685 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767660.210368:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767660.210376:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767660.210381:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767660.210384:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767660.210390:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767660.210394:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767660.210399:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767660.210407:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767660.210411:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767660.210419:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fe300 x1862022294218880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767685 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767660.210437:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294218880:192.168.202.156@tcp:250:kworker.0 00000100:00000001:3.0:1775767660.210447:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767660.210451:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767660.210457:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fe300 x1862022294218880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767685 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767660.210480:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767660.210484:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767660.210489:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b2c00. 02000000:00000001:3.0:1775767660.210495:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767660.210499:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767660.210502:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:3.0:1775767660.210508:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767660.210511:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002bc00. 02000000:00000001:3.0:1775767660.210516:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767660.210519:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767660.210523:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fe300. 00000100:00000001:3.0:1775767660.210528:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767660.210530:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767660.210535:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767660.210542:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767660.210545:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767660.210551:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767660.210554:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767660.210557:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767660.210563:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fd880 x1862022294219008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767660.210619:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767660.210623:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767660.210629:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767660.210632:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767660.210638:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767660.210647:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294219008:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767660.210658:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767660.210662:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767660.210665:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767660.210670:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767660.210675:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b2c00. 02000000:00000001:3.0:1775767660.210679:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767660.210686:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767660.210692:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767660.210699:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767660.210702:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294219008, portal 10 00000100:00000001:3.0:1775767660.210707:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767660.210710:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641202816 : -112966068348800 : ffff9942099fd880) 00000100:00000040:3.0:1775767660.210722:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fd880 x1862022294219008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767685 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767660.210732:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767660.210738:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767660.210743:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767660.210748:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294219008, offset 0 00000400:00000200:3.0:1775767660.210759:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767660.210768:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767660.210777:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767660.210786:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767660.210793:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767660.210799:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767660.210806:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767660.210813:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224884:224884:256:4294967295] 192.168.202.156@tcp LPNI seq info [224880:224880:8:4294967295] 00000400:00000200:3.0:1775767660.210819:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767660.210828:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767660.210836:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767660.210841:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767660.210848:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767660.210851:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775767660.210858:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767660.210862:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000100:3.0:1775767660.210868:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767660.210872:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767660.210877:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767660.210885:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767660.210889:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767660.210893:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767660.210896:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767660.210900:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fd880 x1862022294219008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767685 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767660.210914:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767660.210919:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fd880 x1862022294219008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767685 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767660.210933:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767660.210936:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767660.210941:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767660.210946:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:3.0:1775767660.210950:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767660.210953:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299585043 00000100:00000001:3.0:1775767660.210958:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299586365 00000100:00000001:3.0:1775767660.210962:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299585043 00000100:00000001:3.0:1775767660.210966:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299586365 00000100:00000001:3.0:1775767660.210971:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767660.210976:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767660.210978:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767660.210983:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767660/real 1775767660] req@ffff9942099fd880 x1862022294219008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767660.210996:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767660.210998:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767660.211001:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767660.211004:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767660.211007:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767660.211010:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767660.211014:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fd880 x1862022294219008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767660.211026:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fd880 x1862022294219008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767660.211037:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767660.211040:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767660.211045:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767660.211047:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767660.211052:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fd880 x1862022294219008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767660.211064:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767660.211073:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fd880 x1862022294219008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767685 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767660.211092:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767660.211098:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767660.211101:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767660.211104:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767660.211108:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767660.211113:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767660.211116:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767660.211122:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767660.211126:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767660.211132:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fd880 x1862022294219008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767685 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767660.211144:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294219008:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767660.211150:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767660.211177:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ff480 x1862022294219136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767660.211203:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767660.211205:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767660.211211:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767660.211214:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767660.211218:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767660.211224:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294219136:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767660.211230:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767660.211234:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767660.211237:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767660.211239:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767660.211245:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b0000. 02000000:00000001:3.0:1775767660.211248:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767660.211255:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767660.211259:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767660.211263:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767660.211267:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294219136, portal 10 00000100:00000001:3.0:1775767660.211270:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767660.211273:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641209984 : -112966068341632 : ffff9942099ff480) 00000100:00000040:3.0:1775767660.211281:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ff480 x1862022294219136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767685 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767660.211296:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767660.211302:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767660.211306:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767660.211308:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294219136, offset 0 00000400:00000200:3.0:1775767660.211314:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767660.211322:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767660.211327:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767660.211332:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767660.211337:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767660.211342:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767660.211346:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767660.211350:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224885:224885:256:4294967295] 192.168.202.156@tcp LPNI seq info [224881:224881:8:4294967295] 00000400:00000200:3.0:1775767660.211357:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767660.211365:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767660.211371:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767660.211375:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767660.211382:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767660.211385:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775767660.211391:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767660.211394:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000100:3.0:1775767660.211400:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767660.211402:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767660.211408:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767660.211415:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767660.211420:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767660.211422:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767660.211427:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767660.211432:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ff480 x1862022294219136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767685 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767660.211447:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767660.211451:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ff480 x1862022294219136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767685 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767660.211465:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767660.211468:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767660.211472:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767660.211477:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:3.0:1775767660.211482:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767660.211487:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299585036 00000100:00000001:3.0:1775767660.211491:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299586365 00000100:00000001:3.0:1775767660.211496:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299585036 00000100:00000001:3.0:1775767660.211501:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299586365 00000100:00000001:3.0:1775767660.211505:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767660.211509:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767660.211512:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767660.211517:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767660/real 1775767660] req@ffff9942099ff480 x1862022294219136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767660.211532:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767660.211535:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767660.211537:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767660.211540:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767660.211542:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767660.211545:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767660.211549:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ff480 x1862022294219136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767660.211560:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ff480 x1862022294219136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767660.211572:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767660.211574:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767660.211579:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767660.211583:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767660.211592:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ff480 x1862022294219136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767660.211612:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767660.211622:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ff480 x1862022294219136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767685 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767660.211638:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767660.211643:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767660.211646:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767660.211648:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767660.211650:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767660.211653:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767660.211658:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767660.211663:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767660.211667:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767660.211672:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ff480 x1862022294219136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767685 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767660.211684:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294219136:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767660.211691:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767660.211695:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fd180 x1862022294219264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767660.211703:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767660.211706:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767660.211709:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767660.211711:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767660.211714:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767660.211718:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294219264:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767660.211722:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767660.211725:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767660.211727:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767660.211732:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767660.211735:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b2800. 02000000:00000001:3.0:1775767660.211740:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767660.211749:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767660.211753:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767660.211756:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767660.211760:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294219264, portal 4 00000100:00000001:3.0:1775767660.211762:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767660.211764:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641201024 : -112966068350592 : ffff9942099fd180) 00000100:00000040:3.0:1775767660.211770:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fd180 x1862022294219264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767685 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767660.211777:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767660.211780:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767660.211783:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767660.211785:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294219264, offset 0 00000400:00000200:3.0:1775767660.211789:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767660.211794:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767660.211797:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767660.211800:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767660.211803:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767660.211805:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767660.211807:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767660.211809:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224886:224886:256:4294967295] 192.168.202.156@tcp LPNI seq info [224882:224882:8:4294967295] 00000400:00000200:3.0:1775767660.211812:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767660.211817:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767660.211821:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767660.211823:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767660.211826:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767660.211828:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775767660.211831:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767660.211832:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000100:3.0:1775767660.211835:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767660.211836:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767660.211838:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767660.211840:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767660.211843:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767660.211844:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767660.211846:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767660.211848:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fd180 x1862022294219264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767685 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767660.211854:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767660.211856:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fd180 x1862022294219264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767685 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767660.211860:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767660.211861:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767660.211863:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767660.211865:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:3.0:1775767660.211867:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767660.211869:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299585059 00000100:00000001:3.0:1775767660.211871:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299586365 00000100:00000001:3.0:1775767660.211872:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299585059 00000100:00000001:3.0:1775767660.211874:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299586365 00000100:00000001:3.0:1775767660.211875:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767660.211876:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767660.211878:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767660.211880:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767660/real 1775767660] req@ffff9942099fd180 x1862022294219264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767660.211885:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767660.211886:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767660.211887:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767660.211889:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767660.211890:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767660.211891:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767660.211893:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fd180 x1862022294219264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767660.211899:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fd180 x1862022294219264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767660.211903:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767660.211904:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767660.211905:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767660.211906:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767660.211908:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fd180 x1862022294219264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767660.211912:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767660.211914:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fd180 x1862022294219264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767685 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767660.211918:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767660.211920:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767660.211921:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767660.211922:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767660.211923:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767660.211925:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767660.211926:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767660.211928:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767660.211930:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767660.211932:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fd180 x1862022294219264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767685 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767660.211936:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294219264:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767660.211938:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767660.211940:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767660.211941:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fd880 x1862022294219008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767685 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767660.211946:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767660.211947:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767660.211949:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b2c00. 02000000:00000001:3.0:1775767660.211950:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767660.211951:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767660.211952:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:3.0:1775767660.211954:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767660.211955:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029400. 02000000:00000001:3.0:1775767660.211957:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767660.211958:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767660.211959:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fd880. 00000100:00000001:3.0:1775767660.211960:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767660.211961:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767660.211963:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767660.211964:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ff480 x1862022294219136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767685 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767660.211969:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767660.211969:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767660.211970:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b0000. 02000000:00000001:3.0:1775767660.211972:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767660.211973:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767660.211973:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:3.0:1775767660.211974:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767660.211975:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028000. 02000000:00000001:3.0:1775767660.211976:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767660.211977:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767660.211978:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ff480. 00000100:00000001:3.0:1775767660.211979:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767660.211980:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767660.211981:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767660.211982:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fd180 x1862022294219264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767685 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767660.211986:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767660.211987:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767660.211987:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b2800. 02000000:00000001:3.0:1775767660.211988:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767660.211989:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767660.211990:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:3.0:1775767660.211991:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767660.211992:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028400. 02000000:00000001:3.0:1775767660.211993:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767660.211994:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767660.211995:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fd180. 00000100:00000001:3.0:1775767660.211996:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767660.211997:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767660.211998:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767660.212000:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767660.212001:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767660.212002:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767660.212003:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767660.212004:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767660.212006:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fed80 x1862022294219392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767660.212010:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767660.212011:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767660.212013:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767660.212014:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767660.212015:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767660.212017:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294219392:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767660.212019:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767660.212020:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767660.212021:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767660.212022:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767660.212024:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b2800. 02000000:00000001:3.0:1775767660.212025:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767660.212027:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767660.212028:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767660.212030:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767660.212032:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294219392, portal 4 00000100:00000001:3.0:1775767660.212033:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767660.212034:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641208192 : -112966068343424 : ffff9942099fed80) 00000100:00000040:3.0:1775767660.212037:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fed80 x1862022294219392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767685 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767660.212042:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767660.212043:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767660.212045:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767660.212046:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294219392, offset 0 00000400:00000200:3.0:1775767660.212048:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767660.212051:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767660.212053:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767660.212055:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767660.212057:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767660.212059:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767660.212061:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767660.212064:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224887:224887:256:4294967295] 192.168.202.156@tcp LPNI seq info [224883:224883:8:4294967295] 00000400:00000200:3.0:1775767660.212069:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767660.212075:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767660.212080:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767660.212083:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767660.212087:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767660.212090:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775767660.212094:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767660.212097:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000100:3.0:1775767660.212101:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767660.212103:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767660.212106:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767660.212110:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767660.212113:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767660.212114:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767660.212117:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767660.212120:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fed80 x1862022294219392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767685 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767660.212125:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767660.212127:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fed80 x1862022294219392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767685 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767660.212136:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767660.212137:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767660.212139:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767660.212141:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:3.0:1775767660.212144:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767660.212145:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299585062 00000100:00000001:3.0:1775767660.212147:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299586365 00000100:00000001:3.0:1775767660.212148:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299585062 00000100:00000001:3.0:1775767660.212150:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299586365 00000100:00000001:3.0:1775767660.212151:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767660.212152:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767660.212169:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767660.212172:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767660/real 1775767660] req@ffff9942099fed80 x1862022294219392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767660.212177:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767660.212178:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767660.212192:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767660.212195:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767660.212197:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767660.212198:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767660.212201:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fed80 x1862022294219392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767660.212207:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fed80 x1862022294219392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767660.212211:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767660.212212:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767660.212213:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767660.212214:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767660.212217:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fed80 x1862022294219392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767660.212222:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767660.212224:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fed80 x1862022294219392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767685 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767660.212228:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767660.212230:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767660.212231:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767660.212232:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767660.212233:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767660.212235:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767660.212236:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767660.212238:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767660.212240:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767660.212242:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fed80 x1862022294219392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767685 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767660.212246:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294219392:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767660.212249:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767660.212250:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767660.212252:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fed80 x1862022294219392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767685 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767660.212256:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767660.212257:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767660.212258:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b2800. 02000000:00000001:3.0:1775767660.212259:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767660.212260:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767660.212261:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:3.0:1775767660.212262:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767660.212263:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002a800. 02000000:00000001:3.0:1775767660.212265:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767660.212266:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767660.212267:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fed80. 00000100:00000001:3.0:1775767660.212269:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767660.212270:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767660.212271:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767660.212273:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767660.212274:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767660.212275:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767660.212276:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767660.273305:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775767660.273309:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775767660.273313:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775767661.233238:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767661.233246:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767661.233248:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767661.233255:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767661.233263:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767661.233269:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767661.233284:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767661.233287:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767661.233290:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767662.257334:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767662.257341:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767662.257343:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767662.257349:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767662.257354:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767662.257360:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767662.257364:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767662.257367:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767662.257371:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767663.281231:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767663.281237:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767663.281240:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767663.281246:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767663.281253:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767663.281259:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767663.281271:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767663.281274:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767663.281277:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767664.305264:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767664.305269:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767664.305271:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767664.305276:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767664.305280:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767664.305285:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767664.305288:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767664.305290:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767664.305293:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767665.329268:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767665.329276:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767665.329279:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767665.329286:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767665.329292:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767665.329298:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767665.329325:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767665.329332:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767665.329339:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767665.329566:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767665.329570:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767665.329573:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767665.329581:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207888700 x1862022294219520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767665.329595:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767665.329598:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767665.329603:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767665.329606:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767665.329610:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767665.329619:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207888700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294219520:192.168.202.156@tcp:250:kworker.0 00000100:00000001:3.0:1775767665.329627:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767665.329632:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767665.329634:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767665.329638:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767665.329643:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b2800. 02000000:00000001:3.0:1775767665.329648:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767665.329656:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767665.329661:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767665.329669:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767665.329676:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294219520, portal 25 00000100:00000001:3.0:1775767665.329681:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767665.329684:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606120192 : -112966103431424 : ffff994207888700) 00000100:00000040:3.0:1775767665.329692:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207888700 x1862022294219520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767690 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767665.329702:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767665.329705:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767665.329708:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767665.329711:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294219520, offset 0 00000400:00000200:3.0:1775767665.329718:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767665.329726:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767665.329731:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767665.329736:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767665.329750:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767665.329757:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767665.329762:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767665.329770:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224888:224888:256:4294967295] 192.168.202.156@tcp LPNI seq info [224884:224884:8:4294967295] 00000400:00000200:3.0:1775767665.329776:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767665.329790:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767665.329800:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767665.329806:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767665.329820:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767665.329826:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1775767665.329836:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767665.329840:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000100:3.0:1775767665.329847:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767665.329852:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767665.329858:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767665.329869:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767665.329874:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767665.329878:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767665.329885:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767665.329891:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207888700 x1862022294219520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767690 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767665.329909:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767665.329914:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207888700 x1862022294219520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767690 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767665.329930:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767665.329933:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767665.329938:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767665.329944:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767665.329947:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299591485 00000100:00000010:3.0:1775767665.329953:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207981f00. 00000100:00000001:3.0:1775767665.329962:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1775767665.329968:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299591485 00000100:00000010:3.0:1775767665.329973:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207981180. 00000100:00000001:3.0:1775767665.329979:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1775767665.329985:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767665.329990:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767665.329994:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767665.330000:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767665/real 1775767665] req@ffff994207888700 x1862022294219520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767665.330019:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767665.330023:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767665.330025:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767665.330027:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767665.330028:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767665.330032:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767665.330041:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207888700 x1862022294219520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767665.330074:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207888700 x1862022294219520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767665.330095:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767665.330098:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767665.330104:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767665.330107:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767665.330114:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207888700 x1862022294219520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767665.330127:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767665.330146:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207888700 x1862022294219520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767690 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767665.330178:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767665.330180:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767665.330182:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767665.330183:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767665.330185:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767665.330186:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767665.330189:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767665.330193:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767665.330197:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767665.330203:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207888700 x1862022294219520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767690 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767665.330217:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207888700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294219520:192.168.202.156@tcp:250:kworker.0 00000100:00000001:3.0:1775767665.330230:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767665.330234:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767665.330243:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207888700 x1862022294219520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767690 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767665.330263:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767665.330267:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767665.330272:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b2800. 02000000:00000001:3.0:1775767665.330278:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767665.330282:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767665.330285:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:3.0:1775767665.330291:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767665.330295:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543e800. 02000000:00000001:3.0:1775767665.330298:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767665.330301:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767665.330304:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207888700. 00000100:00000001:3.0:1775767665.330307:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767665.330309:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767665.330313:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767665.330319:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767665.330322:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767665.330324:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767665.330327:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767665.330330:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767665.330337:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207889f80 x1862022294219648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767665.330357:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767665.330360:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767665.330365:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767665.330370:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767665.330376:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767665.330386:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207889f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294219648:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767665.330393:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767665.330398:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767665.330402:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767665.330405:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767665.330409:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b2800. 02000000:00000001:3.0:1775767665.330412:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767665.330439:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767665.330444:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767665.330449:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767665.330454:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294219648, portal 10 00000100:00000001:3.0:1775767665.330459:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767665.330462:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606126464 : -112966103425152 : ffff994207889f80) 00000100:00000040:3.0:1775767665.330473:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207889f80 x1862022294219648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767690 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767665.330482:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767665.330484:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767665.330486:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767665.330488:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294219648, offset 0 00000400:00000200:3.0:1775767665.330492:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767665.330495:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767665.330499:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767665.330501:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767665.330503:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767665.330505:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767665.330506:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767665.330509:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224889:224889:256:4294967295] 192.168.202.156@tcp LPNI seq info [224885:224885:8:4294967295] 00000400:00000200:3.0:1775767665.330512:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767665.330516:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767665.330520:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767665.330523:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767665.330526:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767665.330528:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1775767665.330532:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767665.330534:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000100:3.0:1775767665.330537:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767665.330538:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767665.330540:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767665.330543:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767665.330545:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767665.330546:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767665.330548:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767665.330550:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207889f80 x1862022294219648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767690 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767665.330560:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767665.330562:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207889f80 x1862022294219648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767690 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767665.330566:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767665.330567:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767665.330570:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767665.330572:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:3.0:1775767665.330574:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767665.330576:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299590163 00000100:00000001:3.0:1775767665.330577:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299591485 00000100:00000001:3.0:1775767665.330579:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299590163 00000100:00000001:3.0:1775767665.330580:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299591485 00000100:00000001:3.0:1775767665.330582:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767665.330583:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767665.330584:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767665.330586:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767665/real 1775767665] req@ffff994207889f80 x1862022294219648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767665.330591:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767665.330592:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767665.330593:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767665.330595:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767665.330596:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767665.330597:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767665.330599:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207889f80 x1862022294219648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767665.330604:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207889f80 x1862022294219648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767665.330608:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767665.330610:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767665.330611:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767665.330612:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767665.330614:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207889f80 x1862022294219648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767665.330618:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767665.330621:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207889f80 x1862022294219648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767690 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767665.330625:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767665.330626:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767665.330627:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767665.330628:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767665.330629:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767665.330631:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767665.330632:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767665.330634:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767665.330636:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767665.330638:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207889f80 x1862022294219648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767690 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767665.330643:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207889f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294219648:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767665.330645:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767665.330647:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff99420788b100 x1862022294219776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767665.330652:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767665.330653:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767665.330654:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767665.330655:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767665.330657:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767665.330659:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff99420788b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294219776:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767665.330661:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767665.330662:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767665.330663:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767665.330664:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767665.330667:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b0000. 02000000:00000001:3.0:1775767665.330668:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767665.330670:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767665.330671:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767665.330673:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767665.330675:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294219776, portal 10 00000100:00000001:3.0:1775767665.330676:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767665.330678:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606130944 : -112966103420672 : ffff99420788b100) 00000100:00000040:3.0:1775767665.330681:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff99420788b100 x1862022294219776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767690 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767665.330685:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767665.330686:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767665.330688:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767665.330689:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294219776, offset 0 00000400:00000200:3.0:1775767665.330691:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767665.330693:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767665.330695:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767665.330697:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767665.330699:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767665.330700:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767665.330702:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767665.330704:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224890:224890:256:4294967295] 192.168.202.156@tcp LPNI seq info [224886:224886:8:4294967295] 00000400:00000200:3.0:1775767665.330707:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767665.330711:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767665.330713:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767665.330715:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767665.330718:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767665.330719:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1775767665.330722:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767665.330724:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000100:3.0:1775767665.330726:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767665.330727:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767665.330729:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767665.330731:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767665.330733:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767665.330734:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767665.330735:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767665.330737:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff99420788b100 x1862022294219776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767690 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767665.330742:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767665.330744:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff99420788b100 x1862022294219776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767690 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767665.330748:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767665.330749:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767665.330750:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767665.330752:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:3.0:1775767665.330754:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767665.330755:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299590155 00000100:00000001:3.0:1775767665.330756:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299591485 00000100:00000001:3.0:1775767665.330758:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299590155 00000100:00000001:3.0:1775767665.330759:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299591485 00000100:00000001:3.0:1775767665.330761:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767665.330762:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767665.330763:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767665.330765:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767665/real 1775767665] req@ffff99420788b100 x1862022294219776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767665.330770:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767665.330771:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767665.330772:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767665.330773:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767665.330774:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767665.330776:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767665.330777:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff99420788b100 x1862022294219776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767665.330782:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff99420788b100 x1862022294219776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767665.330786:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767665.330787:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767665.330788:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767665.330788:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767665.330790:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff99420788b100 x1862022294219776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767665.330794:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767665.330796:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff99420788b100 x1862022294219776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767690 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767665.330800:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767665.330802:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767665.330803:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767665.330803:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767665.330804:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767665.330806:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767665.330808:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767665.330810:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767665.330811:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767665.330813:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff99420788b100 x1862022294219776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767690 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767665.330823:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff99420788b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294219776:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767665.330825:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767665.330827:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207888e00 x1862022294219904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767665.330831:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767665.330832:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767665.330833:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767665.330834:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767665.330835:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767665.330837:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207888e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294219904:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767665.330839:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767665.330840:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767665.330841:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767665.330842:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767665.330844:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b2c00. 02000000:00000001:3.0:1775767665.330845:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767665.330847:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767665.330848:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767665.330850:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767665.330852:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294219904, portal 4 00000100:00000001:3.0:1775767665.330854:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767665.330854:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606121984 : -112966103429632 : ffff994207888e00) 00000100:00000040:3.0:1775767665.330857:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207888e00 x1862022294219904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767690 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767665.330861:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767665.330863:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767665.330864:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767665.330865:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294219904, offset 0 00000400:00000200:3.0:1775767665.330868:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767665.330870:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767665.330872:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767665.330874:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767665.330875:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767665.330877:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767665.330879:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767665.330881:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224891:224891:256:4294967295] 192.168.202.156@tcp LPNI seq info [224887:224887:8:4294967295] 00000400:00000200:3.0:1775767665.330883:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767665.330887:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767665.330890:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767665.330891:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767665.330894:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767665.330895:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1775767665.330898:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767665.330899:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000100:3.0:1775767665.330902:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767665.330903:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767665.330905:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767665.330907:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767665.330909:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767665.330910:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767665.330911:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767665.330913:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207888e00 x1862022294219904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767690 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767665.330917:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767665.330919:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207888e00 x1862022294219904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767690 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767665.330923:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767665.330924:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767665.330926:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767665.330927:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:3.0:1775767665.330929:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767665.330930:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299590178 00000100:00000001:3.0:1775767665.330932:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299591485 00000100:00000001:3.0:1775767665.330934:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299590178 00000100:00000001:3.0:1775767665.330935:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299591485 00000100:00000001:3.0:1775767665.330936:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767665.330937:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767665.330939:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767665.330941:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767665/real 1775767665] req@ffff994207888e00 x1862022294219904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767665.330945:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767665.330946:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767665.330947:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767665.330948:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767665.330949:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767665.330950:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767665.330952:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207888e00 x1862022294219904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767665.330957:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207888e00 x1862022294219904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767665.330961:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767665.330962:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767665.330963:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767665.330963:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767665.330965:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207888e00 x1862022294219904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767665.330969:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767665.330971:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207888e00 x1862022294219904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767690 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767665.330975:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767665.330977:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767665.330978:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767665.330979:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767665.330980:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767665.330982:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767665.330984:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767665.330986:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767665.330987:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767665.330989:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207888e00 x1862022294219904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767690 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767665.330994:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207888e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294219904:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767665.330996:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767665.330997:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767665.330999:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207889f80 x1862022294219648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767690 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767665.331003:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767665.331004:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767665.331005:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b2800. 02000000:00000001:3.0:1775767665.331007:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767665.331007:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767665.331008:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:3.0:1775767665.331009:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767665.331010:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543f800. 02000000:00000001:3.0:1775767665.331012:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767665.331013:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767665.331014:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207889f80. 00000100:00000001:3.0:1775767665.331016:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767665.331017:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767665.331018:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767665.331020:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff99420788b100 x1862022294219776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767690 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767665.331028:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767665.331029:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767665.331030:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b0000. 02000000:00000001:3.0:1775767665.331032:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767665.331033:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767665.331034:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:3.0:1775767665.331036:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767665.331037:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543d000. 02000000:00000001:3.0:1775767665.331039:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767665.331040:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767665.331042:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff99420788b100. 00000100:00000001:3.0:1775767665.331044:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767665.331045:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767665.331047:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767665.331050:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207888e00 x1862022294219904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767690 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767665.331057:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767665.331058:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767665.331060:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b2c00. 02000000:00000001:3.0:1775767665.331062:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767665.331063:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767665.331064:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:3.0:1775767665.331066:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767665.331068:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543e400. 02000000:00000001:3.0:1775767665.331070:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767665.331071:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767665.331072:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207888e00. 00000100:00000001:3.0:1775767665.331074:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767665.331076:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767665.331078:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767665.331082:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767665.331083:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767665.331085:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767665.331086:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767665.331088:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767665.331092:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207889880 x1862022294220032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767665.331098:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767665.331099:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767665.331101:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767665.331102:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767665.331103:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767665.331105:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207889880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294220032:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767665.331107:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767665.331109:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767665.331110:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767665.331111:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767665.331113:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b2c00. 02000000:00000001:3.0:1775767665.331115:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767665.331117:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767665.331119:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767665.331121:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767665.331123:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294220032, portal 4 00000100:00000001:3.0:1775767665.331124:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767665.331125:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606124672 : -112966103426944 : ffff994207889880) 00000100:00000040:3.0:1775767665.331129:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207889880 x1862022294220032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767690 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767665.331149:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767665.331152:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767665.331173:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767665.331175:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294220032, offset 0 00000400:00000200:3.0:1775767665.331178:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767665.331181:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767665.331184:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767665.331186:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767665.331188:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767665.331189:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767665.331191:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767665.331194:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224892:224892:256:4294967295] 192.168.202.156@tcp LPNI seq info [224888:224888:8:4294967295] 00000400:00000200:3.0:1775767665.331197:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767665.331201:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767665.331204:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767665.331206:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767665.331209:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767665.331211:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1775767665.331214:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767665.331215:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000100:3.0:1775767665.331218:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767665.331220:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767665.331223:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767665.331227:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767665.331230:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767665.331231:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767665.331234:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767665.331237:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207889880 x1862022294220032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767690 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767665.331245:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767665.331248:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207889880 x1862022294220032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767690 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767665.331255:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767665.331257:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767665.331260:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767665.331263:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:3.0:1775767665.331267:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767665.331269:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299590182 00000100:00000001:3.0:1775767665.331271:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299591485 00000100:00000001:3.0:1775767665.331274:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299590182 00000100:00000001:3.0:1775767665.331276:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299591485 00000100:00000001:3.0:1775767665.331278:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767665.331281:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767665.331283:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767665.331286:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767665/real 1775767665] req@ffff994207889880 x1862022294220032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767665.331294:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767665.331296:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767665.331298:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767665.331299:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767665.331300:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767665.331302:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767665.331304:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207889880 x1862022294220032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767665.331309:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207889880 x1862022294220032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767665.331313:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767665.331314:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767665.331315:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767665.331316:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767665.331318:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207889880 x1862022294220032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767665.331323:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767665.331325:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207889880 x1862022294220032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767690 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767665.331329:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767665.331331:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767665.331332:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767665.331332:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767665.331334:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767665.331337:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767665.331339:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767665.331343:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767665.331345:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767665.331349:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207889880 x1862022294220032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767690 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767665.331357:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207889880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294220032:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767665.331361:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767665.331363:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767665.331366:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207889880 x1862022294220032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767690 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767665.331373:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767665.331375:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767665.331377:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b2c00. 02000000:00000001:3.0:1775767665.331380:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767665.331381:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767665.331382:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:3.0:1775767665.331385:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767665.331386:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543d400. 02000000:00000001:3.0:1775767665.331388:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767665.331390:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767665.331391:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207889880. 00000100:00000001:3.0:1775767665.331393:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767665.331395:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767665.331397:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767665.331399:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767665.331400:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767665.331402:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767665.331403:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767666.353243:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767666.353248:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767666.353256:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767666.353263:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767666.353270:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767666.353276:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767666.353285:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767666.353289:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767666.353294:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767667.377220:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767667.377227:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767667.377232:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767667.377239:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767667.377246:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767667.377251:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767667.377266:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767667.377270:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767667.377273:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767668.401235:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767668.401240:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767668.401242:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767668.401247:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767668.401252:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767668.401255:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767668.401263:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767668.401265:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767668.401268:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767669.425234:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767669.425242:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767669.425245:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767669.425254:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767669.425259:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767669.425265:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767669.425268:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767669.425273:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767669.425279:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767670.449228:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767670.449233:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767670.449236:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767670.449243:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767670.449248:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767670.449252:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767670.449265:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767670.449268:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767670.449272:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767670.450208:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767670.450219:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775767670.450223:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767670.450229:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767670.450235:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767670.450243:0:13471:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 4922 00000400:00000200:3.0:1775767670.450254:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767670.450261:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767670.450264:0:13471:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767670.450268:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767670.450272:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767670.450275:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767670.450278:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767670.450282:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767670.450284:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:3.0:1775767670.450286:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767670.450289:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:3.0:1775767670.450291:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:3.0:1775767670.450294:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:3.0:1775767670.450297:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767670.450299:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767670.450301:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767670.450303:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767670.450306:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:3.0:1775767670.450310:0:13471:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767670.450313:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767670.450315:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767670.450317:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767670.450319:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767670.450322:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775767670.450323:0:13471:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767670.450329:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8380. 00000020:00000040:3.0:1775767670.450332:0:13471:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:3.0:1775767670.450335:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767670.450336:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767670.450339:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767670.450341:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767670.450344:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767670.450349:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2c00. 00000100:00000001:3.0:1775767670.450355:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767670.450362:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8380 x1862022294220160/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767670.450371:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767670.450375:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767670.450378:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055f5621a] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8380 x1862022294220160/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767670.450396:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767670.450398:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767670.450401:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775767670.450402:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767670.450403:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767670.450405:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767670.450407:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4922 00000400:00000200:3.0:1775767670.450411:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767670.450415:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767670.450417:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767670.450419:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767670.450421:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767670.450425:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767670.450429:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767670.450433:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767670.450435:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:3.0:1775767670.450438:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767670.450440:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:3.0:1775767670.450444:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:3.0:1775767670.450450:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:3.0:1775767670.450456:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767670.450460:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767670.450465:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767670.450470:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767670.450474:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00080000:3.0:1775767670.450481:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767670.450486:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767670.450489:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767670.450496:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767670.450500:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767670.450503:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767670.450508:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775767670.450512:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767670.450516:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767670.450524:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8700. 00000020:00000040:3.0:1775767670.450530:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:3.0:1775767670.450536:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767670.450539:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767670.450542:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767670.450546:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767670.450550:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767670.450557:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0000. 00000100:00000001:3.0:1775767670.450562:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767670.450572:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8700 x1862022294220288/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767670.450581:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767670.450584:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767670.450588:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000faa25e3b] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8700 x1862022294220288/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767670.450602:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767670.450607:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767670.450613:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775767670.450615:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767670.450618:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767670.450625:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767670.450632:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4922 00000400:00000200:3.0:1775767670.450641:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767670.450651:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767670.450657:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767670.450663:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767670.450667:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767670.450673:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767670.450679:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767670.450683:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000800:00000200:2.1:1775767670.450683:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000040:3.0:1775767670.450686:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000800:00000200:2.1:1775767670.450687:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000020:00000001:3.0:1775767670.450688:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767670.450692:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:3.0:1775767670.450696:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:3.0:1775767670.450702:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:3.0:1775767670.450707:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767670.450712:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767670.450718:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767670.450722:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767670.450726:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:3.0:1775767670.450732:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767670.450738:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767670.450742:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767670.450747:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767670.450752:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767670.450757:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767670.450760:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775767670.450765:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767670.450769:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767670.450773:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a9180. 00000020:00000040:3.0:1775767670.450778:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:3.0:1775767670.450785:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767670.450788:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767670.450792:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767670.450794:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767670.450796:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767670.450803:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2800. 00000100:00000001:3.0:1775767670.450808:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767670.450816:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a9180 x1862022294220416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767670.450831:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767670.450835:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767670.450843:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bdc2586] to pc [ptlrpcd_rcv+-1] req@ffff9942099a9180 x1862022294220416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767670.450870:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767670.450874:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767670.450879:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775767670.450882:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767670.450884:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767670.450889:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767670.450894:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4922 00000400:00000200:3.0:1775767670.450901:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767670.450908:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767670.450913:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767670.450920:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767670.450924:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767670.450930:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767670.450935:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767670.450939:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767670.450942:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:3.0:1775767670.450946:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767670.450950:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:3.0:1775767670.450955:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:3.0:1775767670.450970:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:3.0:1775767670.450975:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767670.450977:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767670.450982:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767670.450986:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767670.450990:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:3.0:1775767670.450995:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767670.451000:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767670.451005:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767670.451009:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767670.451013:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767670.451015:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767670.451018:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:3.0:1775767670.451023:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767670.451027:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767670.451032:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ab480. 00000020:00000040:3.0:1775767670.451036:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:3.0:1775767670.451043:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767670.451045:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767670.451050:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767670.451053:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767670.451057:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767670.451090:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0c00. 00000100:00000001:3.0:1775767670.451096:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767670.451105:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ab480 x1862022294220544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767670.451119:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767670.451123:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767670.451130:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000067a0ba74] to pc [ptlrpcd_rcv+-1] req@ffff9942099ab480 x1862022294220544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767670.451147:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767670.451186:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767670.451193:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775767670.451196:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767670.451199:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767670.451206:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767670.451212:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4922 00000400:00000200:3.0:1775767670.451219:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767670.451225:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767670.451228:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767670.451234:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767670.451239:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767670.451243:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767670.451249:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767670.451255:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767670.451258:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:3.0:1775767670.451262:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767670.451265:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:3.0:1775767670.451269:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:3.0:1775767670.451275:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:3.0:1775767670.451283:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767670.451289:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767670.451296:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767670.451302:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767670.451308:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:3.0:1775767670.451317:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767670.451323:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767670.451328:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767670.451333:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767670.451339:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767670.451343:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767670.451347:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000800:00000200:2.1:1775767670.451352:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000008:00000001:3.0:1775767670.451353:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775767670.451354:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000100:00000001:3.0:1775767670.451357:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767670.451364:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8e00. 00000020:00000040:3.0:1775767670.451368:0:13471:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:3.0:1775767670.451374:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767670.451377:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767670.451382:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767670.451383:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767670.451387:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767670.451393:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2c00. 00000100:00000001:3.0:1775767670.451398:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767670.451408:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8e00 x1862022294220672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767670.451424:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767670.451429:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767670.451439:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000864a6866] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8e00 x1862022294220672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767670.451461:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767670.451470:0:13471:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (4932) 00000800:00000200:2.1:1775767670.451643:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767670.451644:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1775767670.451921:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767670.451924:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1775767670.452236:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767670.452238:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000100:00000001:3.0:1775767671.473219:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767671.473227:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767671.473232:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767671.473239:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767671.473245:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767671.473250:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767671.473266:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767671.473269:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767671.473272:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767672.497255:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767672.497261:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767672.497265:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767672.497273:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767672.497279:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767672.497284:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767672.497296:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767672.497299:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767672.497303:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767673.521237:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767673.521242:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767673.521244:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767673.521249:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767673.521253:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767673.521258:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767673.521269:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767673.521273:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767673.521280:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767674.545245:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767674.545252:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767674.545256:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767674.545262:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767674.545267:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767674.545275:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767674.545281:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767674.545286:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767674.545293:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767675.569240:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767675.569247:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767675.569252:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767675.569256:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767675.569259:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767675.569261:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767675.569265:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767675.569268:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767675.569271:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767675.569459:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767675.569463:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767675.569465:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767675.569473:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfdc00 x1862022294220800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767675.569487:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767675.569489:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767675.569501:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767675.569504:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767675.569507:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767675.569513:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294220800:192.168.202.156@tcp:250:kworker.0 00000100:00000001:3.0:1775767675.569520:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767675.569525:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767675.569528:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767675.569531:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767675.569536:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b1000. 02000000:00000001:3.0:1775767675.569540:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767675.569549:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767675.569554:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767675.569561:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767675.569567:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294220800, portal 25 00000100:00000001:3.0:1775767675.569571:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767675.569573:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611843584 : -112966097708032 : ffff994207dfdc00) 00000100:00000040:3.0:1775767675.569583:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfdc00 x1862022294220800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767700 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767675.569596:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767675.569601:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767675.569606:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767675.569610:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294220800, offset 0 00000400:00000200:3.0:1775767675.569619:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767675.569626:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767675.569632:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767675.569640:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767675.569648:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767675.569655:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767675.569660:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767675.569668:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224898:224898:256:4294967295] 192.168.202.156@tcp LPNI seq info [224894:224894:8:4294967295] 00000400:00000200:3.0:1775767675.569674:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767675.569689:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767675.569701:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767675.569715:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767675.569727:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767675.569733:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775767675.569743:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767675.569747:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000100:3.0:1775767675.569754:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767675.569757:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767675.569764:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767675.569772:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767675.569778:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767675.569782:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767675.569786:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767675.569791:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfdc00 x1862022294220800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767700 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767675.569814:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767675.569830:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfdc00 x1862022294220800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767700 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767675.569860:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767675.569864:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767675.569874:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767675.569884:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767675.569890:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299601725 00000100:00000010:3.0:1775767675.569900:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942079819c0. 00000100:00000001:3.0:1775767675.569911:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1775767675.569919:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299601725 00000100:00000010:3.0:1775767675.569927:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207981000. 00000100:00000001:3.0:1775767675.569934:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1775767675.569950:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767675.569956:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767675.569962:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767675.569970:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767675/real 1775767675] req@ffff994207dfdc00 x1862022294220800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767675.569987:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767675.569990:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767675.569994:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767675.570001:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767675.570006:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767675.570012:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767675.570022:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfdc00 x1862022294220800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767675.570057:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfdc00 x1862022294220800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767675.570077:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767675.570079:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767675.570083:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767675.570085:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767675.570092:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfdc00 x1862022294220800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767675.570106:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767675.570115:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfdc00 x1862022294220800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767700 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767675.570130:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767675.570135:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767675.570138:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767675.570141:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767675.570146:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767675.570148:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767675.570151:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767675.570173:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767675.570175:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767675.570180:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfdc00 x1862022294220800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767700 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767675.570194:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294220800:192.168.202.156@tcp:250:kworker.0 00000100:00000001:3.0:1775767675.570204:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767675.570207:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767675.570212:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfdc00 x1862022294220800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767700 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767675.570241:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767675.570244:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767675.570246:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b1000. 02000000:00000001:3.0:1775767675.570250:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767675.570252:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767675.570253:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:3.0:1775767675.570254:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767675.570256:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1c800. 02000000:00000001:3.0:1775767675.570259:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767675.570261:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767675.570262:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfdc00. 00000100:00000001:3.0:1775767675.570264:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767675.570265:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767675.570267:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767675.570272:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767675.570273:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767675.570274:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767675.570276:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767675.570277:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767675.570279:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dff100 x1862022294220928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767675.570284:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767675.570285:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767675.570286:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767675.570287:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767675.570289:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767675.570291:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294220928:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767675.570293:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767675.570294:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767675.570295:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767675.570296:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767675.570299:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b1000. 02000000:00000001:3.0:1775767675.570300:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767675.570303:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767675.570305:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767675.570308:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767675.570310:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294220928, portal 10 00000100:00000001:3.0:1775767675.570311:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767675.570312:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611848960 : -112966097702656 : ffff994207dff100) 00000100:00000040:3.0:1775767675.570317:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dff100 x1862022294220928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767700 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767675.570321:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767675.570323:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767675.570324:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767675.570326:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294220928, offset 0 00000400:00000200:3.0:1775767675.570329:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767675.570332:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767675.570335:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767675.570337:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767675.570339:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767675.570341:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767675.570342:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767675.570344:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224899:224899:256:4294967295] 192.168.202.156@tcp LPNI seq info [224895:224895:8:4294967295] 00000400:00000200:3.0:1775767675.570347:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767675.570352:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767675.570354:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767675.570356:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767675.570359:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767675.570361:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775767675.570365:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767675.570366:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000100:3.0:1775767675.570369:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767675.570370:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767675.570372:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767675.570375:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767675.570377:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767675.570378:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767675.570380:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767675.570382:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dff100 x1862022294220928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767700 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767675.570387:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767675.570389:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dff100 x1862022294220928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767700 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767675.570393:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767675.570394:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767675.570396:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767675.570398:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:3.0:1775767675.570400:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767675.570402:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299600403 00000100:00000001:3.0:1775767675.570404:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299601725 00000100:00000001:3.0:1775767675.570405:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299600403 00000100:00000001:3.0:1775767675.570407:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299601725 00000100:00000001:3.0:1775767675.570408:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767675.570410:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767675.570411:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767675.570413:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767675/real 1775767675] req@ffff994207dff100 x1862022294220928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767675.570417:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767675.570418:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767675.570419:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767675.570421:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767675.570422:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767675.570423:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767675.570425:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dff100 x1862022294220928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767675.570430:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dff100 x1862022294220928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767675.570434:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767675.570435:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767675.570436:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767675.570437:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767675.570439:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dff100 x1862022294220928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767675.570443:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767675.570445:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dff100 x1862022294220928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767700 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767675.570449:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767675.570451:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767675.570452:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767675.570453:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767675.570454:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767675.570456:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767675.570458:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767675.570460:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767675.570462:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767675.570464:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dff100 x1862022294220928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767700 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767675.570469:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294220928:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767675.570472:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767675.570474:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfc700 x1862022294221056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767675.570477:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767675.570478:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767675.570479:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767675.570480:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767675.570482:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767675.570483:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294221056:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767675.570485:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767675.570486:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767675.570487:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767675.570488:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767675.570490:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b3400. 02000000:00000001:3.0:1775767675.570491:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767675.570493:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767675.570494:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767675.570496:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767675.570498:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294221056, portal 10 00000100:00000001:3.0:1775767675.570500:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767675.570501:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611838208 : -112966097713408 : ffff994207dfc700) 00000100:00000040:3.0:1775767675.570504:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfc700 x1862022294221056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767700 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767675.570509:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767675.570510:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767675.570511:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767675.570513:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294221056, offset 0 00000400:00000200:3.0:1775767675.570515:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767675.570517:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767675.570519:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767675.570521:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767675.570522:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767675.570524:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767675.570525:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767675.570527:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224900:224900:256:4294967295] 192.168.202.156@tcp LPNI seq info [224896:224896:8:4294967295] 00000400:00000200:3.0:1775767675.570530:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767675.570534:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767675.570537:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767675.570539:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767675.570541:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767675.570543:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775767675.570545:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767675.570547:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000100:3.0:1775767675.570549:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767675.570550:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767675.570552:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767675.570554:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767675.570556:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767675.570557:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767675.570558:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767675.570560:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfc700 x1862022294221056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767700 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767675.570569:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767675.570571:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfc700 x1862022294221056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767700 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767675.570575:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767675.570576:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767675.570578:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767675.570579:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:3.0:1775767675.570581:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767675.570582:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299600395 00000100:00000001:3.0:1775767675.570584:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299601725 00000100:00000001:3.0:1775767675.570585:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299600395 00000100:00000001:3.0:1775767675.570586:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299601725 00000100:00000001:3.0:1775767675.570588:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767675.570589:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767675.570590:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767675.570592:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767675/real 1775767675] req@ffff994207dfc700 x1862022294221056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767675.570597:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767675.570597:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767675.570598:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767675.570600:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767675.570600:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767675.570602:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767675.570603:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfc700 x1862022294221056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767675.570608:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfc700 x1862022294221056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767675.570612:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767675.570614:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767675.570615:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767675.570616:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767675.570618:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfc700 x1862022294221056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767675.570622:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767675.570624:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfc700 x1862022294221056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767700 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767675.570628:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767675.570630:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767675.570631:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767675.570631:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767675.570632:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767675.570634:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767675.570635:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767675.570637:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767675.570638:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767675.570641:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfc700 x1862022294221056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767700 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767675.570645:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294221056:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767675.570647:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767675.570649:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfe300 x1862022294221184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767675.570654:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767675.570655:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767675.570656:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767675.570657:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767675.570658:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767675.570660:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294221184:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767675.570662:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767675.570663:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767675.570664:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767675.570666:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767675.570667:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b0400. 02000000:00000001:3.0:1775767675.570668:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767675.570670:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767675.570672:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767675.570674:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767675.570675:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294221184, portal 4 00000100:00000001:3.0:1775767675.570676:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767675.570678:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611845376 : -112966097706240 : ffff994207dfe300) 00000100:00000040:3.0:1775767675.570681:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfe300 x1862022294221184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767700 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767675.570685:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767675.570686:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767675.570687:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767675.570689:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294221184, offset 0 00000400:00000200:3.0:1775767675.570691:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767675.570693:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767675.570695:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767675.570696:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767675.570698:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767675.570699:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767675.570701:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767675.570703:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224901:224901:256:4294967295] 192.168.202.156@tcp LPNI seq info [224897:224897:8:4294967295] 00000400:00000200:3.0:1775767675.570705:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767675.570709:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767675.570712:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767675.570713:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767675.570716:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767675.570718:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775767675.570720:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767675.570722:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000100:3.0:1775767675.570725:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767675.570726:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767675.570728:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767675.570730:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767675.570731:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767675.570732:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767675.570733:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767675.570735:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfe300 x1862022294221184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767700 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767675.570740:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767675.570742:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfe300 x1862022294221184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767700 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767675.570746:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767675.570747:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767675.570749:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767675.570751:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:3.0:1775767675.570753:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767675.570754:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299600418 00000100:00000001:3.0:1775767675.570756:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299601725 00000100:00000001:3.0:1775767675.570757:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299600418 00000100:00000001:3.0:1775767675.570759:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299601725 00000100:00000001:3.0:1775767675.570760:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767675.570761:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767675.570762:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767675.570764:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767675/real 1775767675] req@ffff994207dfe300 x1862022294221184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767675.570768:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767675.570770:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767675.570771:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767675.570772:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767675.570773:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767675.570774:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767675.570776:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfe300 x1862022294221184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767675.570781:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfe300 x1862022294221184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767675.570784:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767675.570785:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767675.570786:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767675.570787:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767675.570789:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfe300 x1862022294221184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767675.570793:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767675.570795:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfe300 x1862022294221184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767700 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767675.570799:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767675.570800:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767675.570801:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767675.570802:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767675.570803:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767675.570806:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767675.570807:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767675.570809:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767675.570810:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767675.570812:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfe300 x1862022294221184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767700 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767675.570821:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294221184:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767675.570823:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767675.570825:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfed80 x1862022294221312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767675.570829:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767675.570830:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767675.570831:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767675.570832:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767675.570833:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767675.570834:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294221312:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767675.570836:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767675.570837:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767675.570838:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767675.570839:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767675.570841:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b2000. 02000000:00000001:3.0:1775767675.570842:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767675.570844:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767675.570845:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767675.570847:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767675.570848:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294221312, portal 4 00000100:00000001:3.0:1775767675.570850:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767675.570851:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611848064 : -112966097703552 : ffff994207dfed80) 00000100:00000040:3.0:1775767675.570854:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfed80 x1862022294221312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767700 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767675.570858:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767675.570859:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767675.570861:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767675.570862:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294221312, offset 0 00000400:00000200:3.0:1775767675.570864:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767675.570866:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767675.570868:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767675.570870:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767675.570872:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767675.570873:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767675.570875:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767675.570877:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224902:224902:256:4294967295] 192.168.202.156@tcp LPNI seq info [224898:224898:8:4294967295] 00000400:00000200:3.0:1775767675.570879:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767675.570883:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767675.570886:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767675.570887:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767675.570889:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767675.570891:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775767675.570893:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767675.570895:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000100:3.0:1775767675.570897:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767675.570899:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767675.570900:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767675.570904:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767675.570906:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767675.570908:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767675.570910:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767675.570914:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfed80 x1862022294221312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767700 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767675.570922:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767675.570925:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfed80 x1862022294221312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767700 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767675.570931:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767675.570933:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767675.570936:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767675.570939:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:3.0:1775767675.570942:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767675.570945:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299600421 00000100:00000001:3.0:1775767675.570947:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299601725 00000100:00000001:3.0:1775767675.570950:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299600421 00000100:00000001:3.0:1775767675.570952:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299601725 00000100:00000001:3.0:1775767675.570954:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767675.570956:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767675.570958:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767675.570962:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767675/real 1775767675] req@ffff994207dfed80 x1862022294221312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767675.570969:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767675.570970:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767675.570971:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767675.570973:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767675.570974:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767675.570975:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767675.570977:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfed80 x1862022294221312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767675.570985:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfed80 x1862022294221312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767675.570992:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767675.570993:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767675.570995:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767675.570996:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767675.571000:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfed80 x1862022294221312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767675.571006:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767675.571009:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfed80 x1862022294221312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767700 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767675.571015:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767675.571018:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767675.571019:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767675.571020:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767675.571022:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767675.571025:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767675.571027:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767675.571030:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767675.571044:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767675.571047:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfed80 x1862022294221312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767700 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767675.571054:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294221312:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767675.571059:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767675.571060:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767675.571064:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dff100 x1862022294220928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767700 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767675.571070:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767675.571071:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767675.571073:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b1000. 02000000:00000001:3.0:1775767675.571075:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767675.571077:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767675.571078:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:3.0:1775767675.571080:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767675.571081:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1e800. 02000000:00000001:3.0:1775767675.571084:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767675.571085:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767675.571086:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dff100. 00000100:00000001:3.0:1775767675.571089:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767675.571090:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767675.571092:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767675.571095:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfc700 x1862022294221056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767700 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767675.571107:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767675.571108:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767675.571110:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b3400. 02000000:00000001:3.0:1775767675.571112:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767675.571113:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767675.571114:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:3.0:1775767675.571116:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767675.571117:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1f400. 02000000:00000001:3.0:1775767675.571119:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767675.571120:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767675.571121:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfc700. 00000100:00000001:3.0:1775767675.571123:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767675.571124:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767675.571125:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767675.571128:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfe300 x1862022294221184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767700 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767675.571133:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767675.571134:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767675.571136:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b0400. 02000000:00000001:3.0:1775767675.571138:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767675.571139:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767675.571140:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:3.0:1775767675.571142:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767675.571143:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1ec00. 02000000:00000001:3.0:1775767675.571145:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767675.571146:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767675.571147:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfe300. 00000100:00000001:3.0:1775767675.571148:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767675.571149:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767675.571151:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767675.571153:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfed80 x1862022294221312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767700 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767675.571176:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767675.571177:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767675.571179:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b2000. 02000000:00000001:3.0:1775767675.571181:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767675.571183:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767675.571184:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:3.0:1775767675.571185:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767675.571187:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1f000. 02000000:00000001:3.0:1775767675.571189:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767675.571189:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767675.571191:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfed80. 00000100:00000001:3.0:1775767675.571193:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767675.571194:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767675.571196:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767675.571199:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767675.571200:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767675.571202:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767675.571203:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767680.689292:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767680.689303:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767680.689308:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767680.689313:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767680.689320:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767680.689326:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767680.689334:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767680.689340:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767680.689345:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767680.689543:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767680.689549:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767680.689552:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767680.689579:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fc380 x1862022294221440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767680.689612:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767680.689615:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767680.689621:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767680.689628:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767680.689636:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767680.689643:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294221440:192.168.202.156@tcp:250:kworker.0 00000100:00000001:3.0:1775767680.689658:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767680.689666:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767680.689669:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767680.689679:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767680.689688:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b2000. 02000000:00000001:3.0:1775767680.689692:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767680.689705:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767680.689713:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767680.689726:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767680.689736:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294221440, portal 25 00000100:00000001:3.0:1775767680.689745:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767680.689750:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641197440 : -112966068354176 : ffff9942099fc380) 00000100:00000040:3.0:1775767680.689764:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fc380 x1862022294221440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767705 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767680.689816:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767680.689831:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767680.689835:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767680.689846:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294221440, offset 0 00000400:00000200:3.0:1775767680.689870:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767680.689896:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767680.689910:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767680.689923:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767680.689931:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767680.689941:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767680.689949:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767680.689965:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224903:224903:256:4294967295] 192.168.202.156@tcp LPNI seq info [224899:224899:8:4294967295] 00000400:00000200:3.0:1775767680.690002:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767680.690033:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767680.690051:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767680.690061:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767680.690081:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767680.690093:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775767680.690117:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767680.690124:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1775767680.690142:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767680.690147:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767680.690200:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767680.690217:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767680.690230:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767680.690235:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767680.690241:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767680.690251:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fc380 x1862022294221440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767705 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767680.690285:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767680.690296:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fc380 x1862022294221440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767705 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767680.690325:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767680.690332:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767680.690343:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767680.690357:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767680.690360:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299606846 00000100:00000010:3.0:1775767680.690369:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207981720. 00000100:00000001:3.0:1775767680.690383:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1775767680.690392:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299606846 00000100:00000010:3.0:1775767680.690396:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942079815a0. 00000100:00000001:3.0:1775767680.690407:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1775767680.690417:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767680.690427:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767680.690437:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767680.690451:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767680/real 1775767680] req@ffff9942099fc380 x1862022294221440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767680.690487:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767680.690491:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767680.690495:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767680.690501:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767680.690505:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767680.690511:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767680.690521:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fc380 x1862022294221440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767680.690547:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fc380 x1862022294221440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767680.690581:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767680.690586:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767680.690596:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767680.690599:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767680.690607:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fc380 x1862022294221440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767680.690671:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767680.690683:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fc380 x1862022294221440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767705 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767680.690693:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767680.690696:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767680.690698:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767680.690699:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767680.690701:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767680.690704:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767680.690707:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767680.690710:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767680.690712:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767680.690715:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fc380 x1862022294221440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767705 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767680.690720:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294221440:192.168.202.156@tcp:250:kworker.0 00000100:00000001:3.0:1775767680.690726:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767680.690727:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767680.690729:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fc380 x1862022294221440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767705 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767680.690735:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767680.690737:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767680.690739:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b2000. 02000000:00000001:3.0:1775767680.690741:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767680.690743:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767680.690744:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:3.0:1775767680.690746:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767680.690748:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029800. 02000000:00000001:3.0:1775767680.690752:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767680.690753:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767680.690755:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fc380. 00000100:00000001:3.0:1775767680.690758:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767680.690759:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767680.690761:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767680.690769:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767680.690771:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767680.690774:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767680.690776:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767680.690777:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767680.690781:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fca80 x1862022294221568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767680.690789:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767680.690791:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767680.690794:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767680.690796:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767680.690799:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767680.690802:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294221568:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767680.690806:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767680.690809:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767680.690811:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767680.690813:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767680.690817:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b2000. 02000000:00000001:3.0:1775767680.690819:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767680.690823:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767680.690825:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767680.690828:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767680.690830:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294221568, portal 10 00000100:00000001:3.0:1775767680.690832:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767680.690833:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641199232 : -112966068352384 : ffff9942099fca80) 00000100:00000040:3.0:1775767680.690839:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fca80 x1862022294221568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767705 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767680.690844:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767680.690845:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767680.690847:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767680.690849:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294221568, offset 0 00000400:00000200:3.0:1775767680.690852:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767680.690856:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767680.690860:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767680.690864:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767680.690866:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767680.690869:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767680.690872:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767680.690875:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224904:224904:256:4294967295] 192.168.202.156@tcp LPNI seq info [224900:224900:8:4294967295] 00000400:00000200:3.0:1775767680.690880:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767680.690888:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767680.690894:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767680.690898:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767680.690902:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767680.690905:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775767680.690911:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767680.690913:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1775767680.690915:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767680.690917:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767680.690920:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767680.690922:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767680.690925:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767680.690926:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767680.690928:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767680.690931:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fca80 x1862022294221568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767705 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767680.690937:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767680.690939:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fca80 x1862022294221568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767705 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767680.690943:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767680.690944:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767680.690947:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767680.690950:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:3.0:1775767680.690953:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767680.690954:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299605523 00000100:00000001:3.0:1775767680.690956:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299606846 00000100:00000001:3.0:1775767680.690958:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299605523 00000100:00000001:3.0:1775767680.690960:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299606846 00000100:00000001:3.0:1775767680.690961:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767680.690962:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767680.690964:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767680.690967:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767680/real 1775767680] req@ffff9942099fca80 x1862022294221568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767680.690975:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767680.690976:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767680.690978:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767680.690981:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767680.690998:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767680.691001:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767680.691006:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fca80 x1862022294221568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767680.691015:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fca80 x1862022294221568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767680.691023:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767680.691025:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767680.691027:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767680.691028:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767680.691032:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fca80 x1862022294221568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767680.691038:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767680.691041:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fca80 x1862022294221568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767705 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767680.691046:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767680.691048:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767680.691049:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767680.691050:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767680.691051:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767680.691054:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767680.691055:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767680.691057:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767680.691059:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767680.691061:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fca80 x1862022294221568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767705 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767680.691066:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294221568:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767680.691070:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767680.691072:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fea00 x1862022294221696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767680.691081:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767680.691082:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767680.691084:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767680.691086:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767680.691087:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767680.691089:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294221696:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767680.691091:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767680.691092:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767680.691093:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767680.691095:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767680.691097:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b0400. 02000000:00000001:3.0:1775767680.691099:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767680.691101:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767680.691103:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767680.691106:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767680.691108:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294221696, portal 10 00000100:00000001:3.0:1775767680.691109:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767680.691111:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641207296 : -112966068344320 : ffff9942099fea00) 00000100:00000040:3.0:1775767680.691115:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fea00 x1862022294221696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767705 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767680.691119:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767680.691121:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767680.691122:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767680.691124:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294221696, offset 0 00000400:00000200:3.0:1775767680.691127:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767680.691129:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767680.691132:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767680.691134:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767680.691136:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767680.691137:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767680.691139:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767680.691142:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224905:224905:256:4294967295] 192.168.202.156@tcp LPNI seq info [224901:224901:8:4294967295] 00000400:00000200:3.0:1775767680.691145:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767680.691150:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767680.691153:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767680.691174:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767680.691177:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767680.691179:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775767680.691182:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767680.691183:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1775767680.691186:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767680.691187:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767680.691190:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767680.691192:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767680.691194:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767680.691195:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767680.691197:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767680.691200:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fea00 x1862022294221696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767705 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767680.691207:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767680.691210:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fea00 x1862022294221696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767705 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767680.691217:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767680.691219:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767680.691222:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767680.691225:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:3.0:1775767680.691229:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767680.691231:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299605516 00000100:00000001:3.0:1775767680.691234:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299606846 00000100:00000001:3.0:1775767680.691236:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299605516 00000100:00000001:3.0:1775767680.691238:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299606846 00000100:00000001:3.0:1775767680.691240:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767680.691243:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767680.691245:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767680.691250:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767680/real 1775767680] req@ffff9942099fea00 x1862022294221696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767680.691259:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767680.691261:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767680.691263:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767680.691266:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767680.691267:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767680.691270:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767680.691273:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fea00 x1862022294221696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767680.691280:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fea00 x1862022294221696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767680.691285:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767680.691286:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767680.691287:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767680.691288:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767680.691290:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fea00 x1862022294221696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767680.691295:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767680.691297:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fea00 x1862022294221696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767705 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767680.691302:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767680.691303:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767680.691304:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767680.691305:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767680.691306:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767680.691308:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767680.691310:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767680.691312:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767680.691314:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767680.691317:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fea00 x1862022294221696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767705 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767680.691322:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294221696:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767680.691324:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767680.691326:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fe680 x1862022294221824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767680.691331:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767680.691331:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767680.691333:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767680.691334:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767680.691336:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767680.691338:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294221824:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767680.691340:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767680.691342:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767680.691343:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767680.691344:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767680.691347:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b3400. 02000000:00000001:3.0:1775767680.691348:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767680.691351:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767680.691353:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767680.691356:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767680.691358:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294221824, portal 4 00000100:00000001:3.0:1775767680.691359:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767680.691360:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641206400 : -112966068345216 : ffff9942099fe680) 00000100:00000040:3.0:1775767680.691364:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fe680 x1862022294221824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767705 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767680.691369:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767680.691371:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767680.691372:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767680.691375:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294221824, offset 0 00000400:00000200:3.0:1775767680.691377:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767680.691380:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767680.691383:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767680.691385:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767680.691387:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767680.691389:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767680.691390:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767680.691393:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224906:224906:256:4294967295] 192.168.202.156@tcp LPNI seq info [224902:224902:8:4294967295] 00000400:00000200:3.0:1775767680.691396:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767680.691400:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767680.691405:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767680.691408:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767680.691412:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767680.691415:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775767680.691419:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767680.691422:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1775767680.691426:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767680.691427:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767680.691431:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767680.691434:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767680.691437:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767680.691439:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767680.691442:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767680.691446:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fe680 x1862022294221824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767705 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767680.691455:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767680.691458:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fe680 x1862022294221824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767705 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767680.691466:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767680.691468:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767680.691472:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767680.691475:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:3.0:1775767680.691478:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767680.691480:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299605539 00000100:00000001:3.0:1775767680.691482:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299606846 00000100:00000001:3.0:1775767680.691484:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299605539 00000100:00000001:3.0:1775767680.691486:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299606846 00000100:00000001:3.0:1775767680.691487:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767680.691489:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767680.691490:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767680.691492:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767680/real 1775767680] req@ffff9942099fe680 x1862022294221824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767680.691497:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767680.691498:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767680.691499:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767680.691501:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767680.691502:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767680.691504:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767680.691506:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fe680 x1862022294221824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767680.691511:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fe680 x1862022294221824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767680.691516:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767680.691517:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767680.691518:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767680.691519:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767680.691521:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fe680 x1862022294221824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767680.691525:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767680.691527:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fe680 x1862022294221824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767705 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767680.691532:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767680.691534:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767680.691535:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767680.691536:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767680.691537:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767680.691540:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767680.691542:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767680.691544:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767680.691545:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767680.691547:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fe680 x1862022294221824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767705 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767680.691552:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294221824:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767680.691554:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767680.691557:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fdc00 x1862022294221952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767680.691561:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767680.691563:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767680.691564:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767680.691566:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767680.691567:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767680.691569:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294221952:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767680.691571:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767680.691573:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767680.691574:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767680.691575:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767680.691578:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b1000. 02000000:00000001:3.0:1775767680.691579:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767680.691581:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767680.691583:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767680.691585:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767680.691587:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294221952, portal 4 00000100:00000001:3.0:1775767680.691588:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767680.691589:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641203712 : -112966068347904 : ffff9942099fdc00) 00000100:00000040:3.0:1775767680.691594:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fdc00 x1862022294221952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767705 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767680.691598:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767680.691600:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767680.691601:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767680.691603:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294221952, offset 0 00000400:00000200:3.0:1775767680.691605:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767680.691608:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767680.691610:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767680.691612:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767680.691614:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767680.691615:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767680.691617:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767680.691619:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224907:224907:256:4294967295] 192.168.202.156@tcp LPNI seq info [224903:224903:8:4294967295] 00000400:00000200:3.0:1775767680.691622:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767680.691626:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767680.691629:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767680.691630:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767680.691633:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767680.691635:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775767680.691638:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767680.691640:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1775767680.691643:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767680.691644:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767680.691646:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767680.691648:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767680.691650:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767680.691652:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767680.691653:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767680.691656:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fdc00 x1862022294221952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767705 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767680.691661:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767680.691663:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fdc00 x1862022294221952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767705 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767680.691667:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767680.691668:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767680.691670:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767680.691673:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:3.0:1775767680.691674:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767680.691675:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299605542 00000100:00000001:3.0:1775767680.691677:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299606846 00000100:00000001:3.0:1775767680.691679:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299605542 00000100:00000001:3.0:1775767680.691680:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299606846 00000100:00000001:3.0:1775767680.691682:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767680.691683:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767680.691684:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767680.691686:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767680/real 1775767680] req@ffff9942099fdc00 x1862022294221952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767680.691690:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767680.691691:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767680.691692:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767680.691693:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767680.691694:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767680.691695:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767680.691697:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fdc00 x1862022294221952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767680.691702:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fdc00 x1862022294221952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767680.691710:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767680.691712:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767680.691713:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767680.691714:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767680.691716:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fdc00 x1862022294221952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767680.691720:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767680.691722:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fdc00 x1862022294221952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767705 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767680.691726:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767680.691727:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767680.691728:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767680.691729:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767680.691730:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767680.691732:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767680.691734:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767680.691736:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767680.691737:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767680.691739:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fdc00 x1862022294221952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767705 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767680.691744:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294221952:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767680.691746:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767680.691747:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767680.691749:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fca80 x1862022294221568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767705 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767680.691754:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767680.691755:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767680.691757:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b2000. 02000000:00000001:3.0:1775767680.691758:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767680.691759:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767680.691761:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:3.0:1775767680.691763:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767680.691765:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002a800. 02000000:00000001:3.0:1775767680.691767:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767680.691769:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767680.691770:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fca80. 00000100:00000001:3.0:1775767680.691773:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767680.691775:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767680.691776:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767680.691780:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fea00 x1862022294221696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767705 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767680.691787:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767680.691788:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767680.691790:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b0400. 02000000:00000001:3.0:1775767680.691793:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767680.691795:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767680.691796:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:3.0:1775767680.691798:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767680.691800:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028400. 02000000:00000001:3.0:1775767680.691803:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767680.691804:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767680.691805:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fea00. 00000100:00000001:3.0:1775767680.691808:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767680.691810:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767680.691812:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767680.691815:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fe680 x1862022294221824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767705 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767680.691822:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767680.691823:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767680.691824:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b3400. 02000000:00000001:3.0:1775767680.691826:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767680.691827:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767680.691828:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:3.0:1775767680.691830:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767680.691831:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028000. 02000000:00000001:3.0:1775767680.691832:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767680.691833:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767680.691834:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fe680. 00000100:00000001:3.0:1775767680.691835:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767680.691836:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767680.691837:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767680.691839:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fdc00 x1862022294221952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767705 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767680.691843:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767680.691843:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767680.691844:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b1000. 02000000:00000001:3.0:1775767680.691845:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767680.691846:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767680.691847:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:3.0:1775767680.691848:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767680.691849:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029400. 02000000:00000001:3.0:1775767680.691850:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767680.691850:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767680.691851:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fdc00. 00000100:00000001:3.0:1775767680.691853:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767680.691854:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767680.691856:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767680.691860:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767680.691861:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767680.691863:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767680.691865:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767681.713291:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767681.713299:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767681.713301:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767681.713309:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767681.713317:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767681.713323:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767681.713336:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767681.713338:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767681.713341:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767682.737227:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767682.737234:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767682.737238:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767682.737246:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767682.737253:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767682.737257:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767682.737261:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767682.737264:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767682.737266:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767683.761207:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767683.761212:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767683.761214:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767683.761221:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767683.761226:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767683.761229:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767683.761231:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767683.761234:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767683.761237:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767683.761277:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767683.761285:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767683.761293:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767683.761310:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767683.761324:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767683.761329:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767683.761351:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767683.761354:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767683.761358:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767684.785223:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767684.785229:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767684.785231:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767684.785236:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767684.785242:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767684.785246:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767684.785250:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767684.785257:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767684.785262:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767685.809221:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767685.809230:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767685.809233:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767685.809239:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767685.809244:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767685.809249:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767685.809253:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767685.809256:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767685.809262:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767685.809301:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767685.809310:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775767685.809314:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767685.809318:0:36:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767685.809323:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767685.809331:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 4938 00000400:00000200:3.0:1775767685.809342:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767685.809349:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767685.809353:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767685.809358:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767685.809362:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767685.809366:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767685.809369:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767685.809374:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767685.809376:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:3.0:1775767685.809387:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767685.809391:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:3.0:1775767685.809398:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:3.0:1775767685.809403:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:3.0:1775767685.809409:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767685.809414:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767685.809419:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767685.809424:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767685.809430:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:3.0:1775767685.809437:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767685.809444:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767685.809451:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767685.809455:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767685.809458:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767685.809461:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775767685.809464:0:36:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767685.809471:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aad80. 00000020:00000040:3.0:1775767685.809476:0:36:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:3.0:1775767685.809481:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767685.809483:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767685.809487:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767685.809491:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767685.809496:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767685.809505:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b1000. 00000100:00000001:3.0:1775767685.809513:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767685.809525:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aad80 x1862022294222080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767685.809539:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767685.809545:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767685.809550:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000045ccf08] to pc [ptlrpcd_rcv+-1] req@ffff9942099aad80 x1862022294222080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767685.809575:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767685.809581:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767685.809587:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775767685.809590:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767685.809593:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767685.809599:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767685.809604:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4938 00000400:00000200:3.0:1775767685.809612:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767685.809619:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767685.809625:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767685.809639:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767685.809648:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767685.809654:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767685.809658:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767685.809663:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767685.809666:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:3.0:1775767685.809670:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767685.809674:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:3.0:1775767685.809679:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:3.0:1775767685.809683:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:3.0:1775767685.809690:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767685.809694:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767685.809699:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767685.809701:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767685.809704:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00080000:3.0:1775767685.809708:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767685.809713:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767685.809716:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767685.809722:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767685.809727:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767685.809729:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767685.809736:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775767685.809741:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767685.809745:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767685.809750:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aa680. 00000020:00000040:3.0:1775767685.809754:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:3.0:1775767685.809758:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767685.809761:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767685.809765:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767685.809768:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767685.809771:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767685.809777:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b3400. 00000100:00000001:3.0:1775767685.809784:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767685.809795:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aa680 x1862022294222208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767685.809813:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767685.809816:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767685.809823:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006cb1597d] to pc [ptlrpcd_rcv+-1] req@ffff9942099aa680 x1862022294222208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767685.809848:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767685.809855:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767685.809867:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775767685.809870:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767685.809874:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767685.809882:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767685.809890:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4938 00000400:00000200:3.0:1775767685.809900:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767685.809909:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767685.809916:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767685.809925:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767685.809931:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767685.809950:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767685.809956:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767685.809963:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767685.809966:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:3.0:1775767685.809971:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767685.809975:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000800:00000200:2.1:1775767685.809979:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000001:3.0:1775767685.809981:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:3.0:1775767685.809987:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000800:00000200:2.1:1775767685.809987:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000100:00000040:3.0:1775767685.809995:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767685.809999:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767685.810005:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767685.810011:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767685.810017:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:3.0:1775767685.810024:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767685.810031:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767685.810037:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767685.810041:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767685.810047:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767685.810051:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767685.810055:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775767685.810059:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767685.810062:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767685.810071:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ab800. 00000020:00000040:3.0:1775767685.810077:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:3.0:1775767685.810084:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767685.810087:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767685.810090:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767685.810093:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767685.810097:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767685.810104:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0400. 00000100:00000001:3.0:1775767685.810111:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767685.810121:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ab800 x1862022294222336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767685.810134:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767685.810137:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767685.810144:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000089df80c0] to pc [ptlrpcd_rcv+-1] req@ffff9942099ab800 x1862022294222336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767685.810198:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767685.810204:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767685.810209:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775767685.810211:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767685.810215:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767685.810221:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767685.810227:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4938 00000400:00000200:3.0:1775767685.810234:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767685.810243:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767685.810248:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767685.810254:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767685.810257:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767685.810262:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767685.810265:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767685.810270:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767685.810273:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:3.0:1775767685.810277:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767685.810280:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:3.0:1775767685.810285:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:3.0:1775767685.810291:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:3.0:1775767685.810300:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767685.810304:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767685.810308:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767685.810311:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767685.810313:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:3.0:1775767685.810318:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767685.810327:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767685.810333:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767685.810337:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767685.810341:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767685.810343:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767685.810347:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:3.0:1775767685.810353:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767685.810357:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767685.810365:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aa300. 00000020:00000040:3.0:1775767685.810371:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:3.0:1775767685.810378:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767685.810382:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767685.810386:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767685.810391:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767685.810396:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767685.810405:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2000. 00000100:00000001:3.0:1775767685.810412:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767685.810421:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aa300 x1862022294222464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767685.810453:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767685.810459:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767685.810467:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f33bdf15] to pc [ptlrpcd_rcv+-1] req@ffff9942099aa300 x1862022294222464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767685.810501:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767685.810507:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767685.810514:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775767685.810517:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767685.810520:0:36:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767685.810526:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767685.810531:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4938 00000400:00000200:3.0:1775767685.810540:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767685.810545:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767685.810550:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767685.810556:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767685.810562:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767685.810566:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767685.810571:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767685.810577:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767685.810581:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:3.0:1775767685.810585:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767685.810589:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:3.0:1775767685.810594:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:3.0:1775767685.810599:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:3.0:1775767685.810607:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767685.810613:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767685.810618:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767685.810623:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767685.810628:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:3.0:1775767685.810635:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767685.810641:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767685.810647:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767685.810652:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767685.810657:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767685.810661:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767685.810665:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:3.0:1775767685.810670:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767685.810675:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767685.810679:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8a80. 00000020:00000040:3.0:1775767685.810685:0:36:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:3.0:1775767685.810690:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767685.810693:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767685.810698:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767685.810701:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767685.810704:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767685.810711:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b1000. 00000100:00000001:3.0:1775767685.810719:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767685.810729:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8a80 x1862022294222592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767685.810748:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767685.810753:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767685.810764:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006c3265af] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8a80 x1862022294222592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767685.810793:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767685.810801:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (4948) 00000800:00000200:2.1:1775767685.810811:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767685.810814:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775767685.811070:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767685.811072:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775767685.811319:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767685.811321:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775767685.811589:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767685.811591:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000100:00000001:3.0:1775767686.833252:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767686.833263:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767686.833267:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767686.833274:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767686.833289:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767686.833298:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767686.833310:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767686.833316:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767686.833321:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767687.857220:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767687.857226:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767687.857228:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767687.857232:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767687.857236:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767687.857240:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767687.857245:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767687.857248:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767687.857254:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767688.881274:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767688.881283:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767688.881287:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767688.881296:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767688.881302:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767688.881320:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767688.881336:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767688.881341:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767688.881346:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767689.905250:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767689.905257:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767689.905260:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767689.905267:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767689.905273:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767689.905277:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767689.905286:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767689.905290:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767689.905295:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767690.929275:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767690.929287:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767690.929294:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767690.929302:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767690.929312:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767690.929321:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767690.929329:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767690.929338:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767690.929350:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775767690.930021:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767690.930027:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000100:00000001:1.0:1775767690.931078:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767690.931084:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775767690.931089:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:2.1:1775767690.931096:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 02000000:00000001:1.0:1775767690.931097:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775767690.931100:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000100:00000001:1.0:1775767690.931102:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775767690.931107:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:1.0:1775767690.931114:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767690.931120:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775767690.931127:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207888700. 00000020:00000040:1.0:1775767690.931134:0:36:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:1.0:1775767690.931142:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775767690.931146:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767690.931152:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775767690.931180:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775767690.931185:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775767690.931193:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543d000. 00000100:00000001:1.0:1775767690.931200:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767690.931212:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207888700 x1862022294223232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775767690.931227:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775767690.931232:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767690.931244:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009f2ba301] to pc [ptlrpcd_rcv+-1] req@ffff994207888700 x1862022294223232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767690.931285:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767690.931294:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (4953) 00000800:00000200:2.1:1775767690.931912:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767690.931920:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775767690.932419:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767690.932421:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775767690.932680:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767690.932682:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000100:00000001:3.0:1775767691.953259:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767691.953266:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767691.953269:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767691.953273:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767691.953277:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767691.953281:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767691.953283:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767691.953286:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767691.953289:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767692.273264:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299617086, at 4299617088 00000100:00000010:1.0:1775767692.273277:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d63c0. 00000100:00000001:3.0:1775767692.977230:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767692.977236:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767692.977240:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767692.977245:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767692.977251:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767692.977255:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767692.977265:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767692.977269:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767692.977274:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767694.001248:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767694.001256:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767694.001260:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767694.001266:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767694.001272:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767694.001275:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767694.001283:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767694.001286:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767694.001288:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767695.025232:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767695.025239:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767695.025242:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767695.025250:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767695.025257:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767695.025262:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767695.025273:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767695.025276:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767695.025279:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767696.049254:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767696.049260:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767696.049263:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767696.049268:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775767696.049269:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767696.049275:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:1.0:1775767696.049278:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775767696.049283:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1775767696.049288:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1775767696.049289:0:36:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767696.049292:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767696.049294:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:3.0:1775767696.049295:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767696.049300:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767696.049304:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 4948 00000400:00000200:1.0:1775767696.049320:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767696.049330:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775767696.049334:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775767696.049338:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767696.049341:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767696.049345:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767696.049347:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775767696.049351:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775767696.049353:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:1.0:1775767696.049356:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775767696.049358:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:1.0:1775767696.049361:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:1.0:1775767696.049363:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:1.0:1775767696.049366:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767696.049368:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767696.049370:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767696.049372:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775767696.049375:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:1.0:1775767696.049379:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775767696.049381:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767696.049384:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775767696.049387:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767696.049390:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767696.049392:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775767696.049395:0:36:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775767696.049405:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207888700. 00000020:00000040:1.0:1775767696.049408:0:36:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:1.0:1775767696.049412:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775767696.049414:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767696.049416:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775767696.049419:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775767696.049423:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775767696.049433:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543f800. 00000100:00000001:1.0:1775767696.049440:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767696.049449:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207888700 x1862022294223360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775767696.049460:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775767696.049464:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767696.049469:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009f2ba301] to pc [ptlrpcd_rcv+-1] req@ffff994207888700 x1862022294223360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767696.049490:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767696.049493:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767696.049497:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775767696.049498:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775767696.049500:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775767696.049505:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775767696.049508:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4948 00000400:00000200:1.0:1775767696.049513:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767696.049519:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775767696.049523:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775767696.049529:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767696.049534:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767696.049539:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767696.049545:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775767696.049550:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775767696.049553:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:1.0:1775767696.049556:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775767696.049559:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:1.0:1775767696.049564:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:1.0:1775767696.049569:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:1.0:1775767696.049575:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767696.049580:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767696.049584:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767696.049589:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775767696.049595:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00080000:1.0:1775767696.049600:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775767696.049604:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767696.049609:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775767696.049614:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767696.049619:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767696.049624:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775767696.049630:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775767696.049636:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767696.049641:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775767696.049648:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207889f80. 00000020:00000040:1.0:1775767696.049654:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:1.0:1775767696.049660:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775767696.049663:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767696.049666:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775767696.049670:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775767696.049673:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775767696.049678:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543e800. 00000100:00000001:1.0:1775767696.049683:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767696.049694:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207889f80 x1862022294223488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775767696.049714:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775767696.049719:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767696.049728:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ed2274f4] to pc [ptlrpcd_rcv+-1] req@ffff994207889f80 x1862022294223488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767696.049753:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767696.049757:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767696.049762:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775767696.049767:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775767696.049771:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:2.1:1775767696.049776:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:1.0:1775767696.049778:0:36:0:(import.c:484:import_select_connection()) Process entered 00000800:00000200:2.1:1775767696.049782:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000100:00080000:1.0:1775767696.049784:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4948 00000400:00000200:1.0:1775767696.049795:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767696.049801:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775767696.049805:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775767696.049813:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767696.049818:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767696.049823:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767696.049827:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775767696.049850:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775767696.049853:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:1.0:1775767696.049857:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775767696.049860:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:1.0:1775767696.049866:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:1.0:1775767696.049872:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:1.0:1775767696.049880:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767696.049884:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767696.049890:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767696.049895:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775767696.049898:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:1.0:1775767696.049904:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775767696.049907:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767696.049912:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775767696.049915:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767696.049919:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767696.049923:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775767696.049926:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775767696.049932:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767696.049937:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775767696.049942:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff99420788b100. 00000020:00000040:1.0:1775767696.049947:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:1.0:1775767696.049954:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775767696.049957:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767696.049960:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775767696.049963:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775767696.049967:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775767696.049974:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543c400. 00000100:00000001:1.0:1775767696.049980:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767696.049990:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff99420788b100 x1862022294223616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775767696.050005:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775767696.050009:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767696.050019:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e26cf977] to pc [ptlrpcd_rcv+-1] req@ffff99420788b100 x1862022294223616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767696.050044:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767696.050050:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767696.050059:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775767696.050064:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775767696.050067:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775767696.050070:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775767696.050077:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4948 00000400:00000200:1.0:1775767696.050084:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767696.050087:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775767696.050090:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775767696.050096:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767696.050101:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767696.050106:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767696.050111:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775767696.050114:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775767696.050118:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:1.0:1775767696.050122:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775767696.050125:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:1.0:1775767696.050128:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:1.0:1775767696.050133:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:1.0:1775767696.050141:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767696.050147:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767696.050151:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767696.050184:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775767696.050189:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:1.0:1775767696.050195:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775767696.050203:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767696.050207:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775767696.050212:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767696.050216:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767696.050220:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775767696.050225:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:1.0:1775767696.050230:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767696.050234:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775767696.050242:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207888e00. 00000020:00000040:1.0:1775767696.050247:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:1.0:1775767696.050254:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775767696.050258:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767696.050262:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775767696.050265:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775767696.050269:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775767696.050276:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543fc00. 00000100:00000001:1.0:1775767696.050283:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767696.050292:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207888e00 x1862022294223744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775767696.050307:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775767696.050311:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767696.050321:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001812f966] to pc [ptlrpcd_rcv+-1] req@ffff994207888e00 x1862022294223744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767696.050349:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767696.050354:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767696.050359:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775767696.050362:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775767696.050365:0:36:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775767696.050371:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775767696.050376:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4948 00000400:00000200:1.0:1775767696.050384:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767696.050394:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775767696.050397:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775767696.050401:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767696.050403:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767696.050408:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767696.050414:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775767696.050419:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775767696.050423:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:1.0:1775767696.050426:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775767696.050430:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:1.0:1775767696.050435:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:1.0:1775767696.050441:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:1.0:1775767696.050449:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767696.050456:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767696.050462:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000800:00000200:2.1:1775767696.050463:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767696.050468:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000100:00000001:1.0:1775767696.050469:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775767696.050473:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:1.0:1775767696.050481:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775767696.050487:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767696.050493:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775767696.050498:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767696.050504:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767696.050510:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775767696.050514:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:1.0:1775767696.050520:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767696.050526:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775767696.050533:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207889880. 00000020:00000040:1.0:1775767696.050539:0:36:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:1.0:1775767696.050545:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775767696.050548:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767696.050553:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775767696.050557:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775767696.050561:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775767696.050569:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543c800. 00000100:00000001:1.0:1775767696.050575:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767696.050586:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207889880 x1862022294223872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775767696.050605:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775767696.050609:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767696.050621:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f11856b8] to pc [ptlrpcd_rcv+-1] req@ffff994207889880 x1862022294223872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767696.050637:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767696.050642:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (4958) 00000800:00000200:2.1:1775767696.050768:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767696.050770:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775767696.051022:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767696.051025:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775767696.051255:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767696.051257:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000100:00000001:3.0:1775767697.073267:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767697.073274:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767697.073276:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767697.073281:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767697.073286:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767697.073291:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767697.073296:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767697.073300:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767697.073302:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767697.393257:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299622205, at 4299622208 00000100:00000010:1.0:1775767697.393273:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d6ea0. 00000100:00000001:1.0:1775767698.097272:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767698.097278:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767698.097281:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767698.097287:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767698.097291:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767698.097295:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767698.097298:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767698.097301:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767698.097304:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767699.121254:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767699.121260:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767699.121263:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767699.121273:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767699.121281:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767699.121286:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767699.121294:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767699.121297:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767699.121301:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767700.145270:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767700.145280:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767700.145285:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767700.145293:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767700.145299:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767700.145303:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767700.145306:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767700.145310:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767700.145314:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767701.169302:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767701.169307:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767701.169310:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767701.169316:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767701.169325:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767701.169330:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767701.169333:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767701.169336:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767701.169339:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767701.169359:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767701.169367:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767701.169371:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767701.169374:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767701.169379:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767701.169382:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767701.169387:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767701.169391:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767701.169395:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775767701.170052:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767701.170592:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767701.171430:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767701.171620:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767701.171763:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:3.0:1775767702.193376:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767702.193377:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767702.193382:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767702.193382:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767702.193386:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767702.193386:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767702.193392:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775767702.193393:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767702.193396:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767702.193398:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767702.193399:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767702.193402:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767702.193402:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767702.193405:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767702.193408:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767702.193408:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767702.193411:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767702.193414:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767702.513293:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299627325, at 4299627328 00000100:00000010:1.0:1775767702.513314:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d6a20. 00000100:00000001:3.0:1775767703.217312:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767703.217325:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767703.217334:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767703.217344:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767703.217350:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767703.217358:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767703.217360:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767703.217366:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767703.217367:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767703.217372:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767703.217374:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767703.217376:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767703.217383:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767703.217385:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767703.217386:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767703.217393:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767703.217396:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767703.217400:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767704.241260:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767704.241271:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767704.241277:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767704.241286:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767704.241297:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767704.241305:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767704.241320:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767704.241323:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767704.241327:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767704.242399:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767704.242405:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767704.242409:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767704.242418:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767701/real 1775767704] req@ffff9942099fea00 x1862022294224384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767704.242434:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767704.242437:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767704.242442:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39948. 00000400:00000200:3.0:1775767704.242447:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8fe58 00000400:00000010:3.0:1775767704.242449:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8fe58. 00000100:00000001:3.0:1775767704.242452:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767704.242456:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fea00 x1862022294224384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767704.242466:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fea00 x1862022294224384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767704.242474:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767704.242476:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767704.242480:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767704.242485:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767704.242490:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fea00 x1862022294224384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767704.242497:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767704.242502:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fea00 x1862022294224384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767726 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767704.242510:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767704.242516:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767704.242518:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767704.242519:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767704.242522:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767704.242529:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767704.242536:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767704.242543:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767704.242546:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767704.242551:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fea00 x1862022294224384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767726 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767704.242559:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294224384:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767704.242565:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767704.242568:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767701/real 1775767704] req@ffff9942099fca80 x1862022294224512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767704.242576:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767704.242577:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767704.242579:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39688. 00000400:00000200:3.0:1775767704.242581:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f220 00000400:00000010:3.0:1775767704.242583:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f220. 00000100:00000001:3.0:1775767704.242585:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767704.242589:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fca80 x1862022294224512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767704.242597:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fca80 x1862022294224512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767704.242604:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767704.242605:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767704.242607:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767704.242609:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767704.242613:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fca80 x1862022294224512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767704.242619:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767704.242623:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fca80 x1862022294224512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767726 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767704.242631:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767704.242633:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767704.242635:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767704.242637:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767704.242639:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767704.242645:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767704.242648:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767704.242652:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767704.242655:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767704.242659:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fca80 x1862022294224512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767726 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767704.242667:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294224512:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767704.242672:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767704.242675:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767701/real 1775767704] req@ffff9942099fe680 x1862022294224256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767704.242692:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767704.242693:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767704.242696:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39c60. 00000400:00000200:3.0:1775767704.242698:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8ff68 00000400:00000010:3.0:1775767704.242700:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8ff68. 00000100:00000001:3.0:1775767704.242702:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767704.242705:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fe680 x1862022294224256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767704.242713:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fe680 x1862022294224256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767704.242720:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767704.242721:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767704.242723:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767704.242725:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767704.242729:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fe680 x1862022294224256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767704.242737:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767704.242741:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fe680 x1862022294224256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767726 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767704.242748:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767704.242750:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767704.242752:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767704.242767:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767704.242769:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767704.242772:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767704.242775:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767704.242779:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767704.242781:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767704.242786:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fe680 x1862022294224256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767726 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767704.242796:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294224256:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767704.242801:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767704.242805:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767701/real 1775767704] req@ffff9942099fdc00 x1862022294224128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767704.242812:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767704.242814:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767704.242816:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39e70. 00000400:00000200:3.0:1775767704.242819:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:3.0:1775767704.242821:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:3.0:1775767704.242823:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767704.242826:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fdc00 x1862022294224128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767704.242836:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fdc00 x1862022294224128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767704.242842:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767704.242843:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767704.242845:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767704.242846:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767704.242849:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fdc00 x1862022294224128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767704.242856:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767704.242859:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fdc00 x1862022294224128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767726 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767704.242865:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767704.242867:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767704.242870:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767704.242871:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767704.242873:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767704.242876:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767704.242879:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767704.242882:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767704.242885:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767704.242889:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fdc00 x1862022294224128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767726 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767704.242896:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294224128:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767704.242901:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767704.242905:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767701/real 1775767704] req@ffff9942099fce00 x1862022294224000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767704.242912:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767704.242913:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767704.242915:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39630. 00000400:00000200:3.0:1775767704.242918:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f440 00000400:00000010:3.0:1775767704.242920:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f440. 00000100:00000001:3.0:1775767704.242922:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767704.242926:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fce00 x1862022294224000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767704.242934:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fce00 x1862022294224000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767704.242940:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767704.242942:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767704.242944:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767704.242945:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767704.242948:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fce00 x1862022294224000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767704.242962:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767704.242966:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fce00 x1862022294224000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767726 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767704.242973:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767704.242975:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767704.242977:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767704.242978:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767704.242980:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767704.242982:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767704.242985:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767704.242988:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767704.242991:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767704.242995:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fce00 x1862022294224000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767726 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767704.243004:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294224000:192.168.202.156@tcp:250:kworker.0 00000100:00000001:3.0:1775767704.243010:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767704.243012:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767704.243015:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fea00 x1862022294224384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767726 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767704.243024:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767704.243028:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767704.243035:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1d400. 02000000:00000001:3.0:1775767704.243041:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767704.243044:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767704.243046:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:3.0:1775767704.243049:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767704.243052:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028000. 02000000:00000001:3.0:1775767704.243057:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767704.243061:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767704.243063:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fea00. 00000100:00000001:3.0:1775767704.243068:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767704.243069:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767704.243071:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767704.243075:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fca80 x1862022294224512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767726 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767704.243081:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767704.243082:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767704.243083:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f800. 02000000:00000001:3.0:1775767704.243086:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767704.243087:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767704.243089:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:3.0:1775767704.243091:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767704.243093:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028400. 02000000:00000001:3.0:1775767704.243095:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767704.243096:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767704.243098:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fca80. 00000100:00000001:3.0:1775767704.243100:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767704.243101:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767704.243103:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767704.243106:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fe680 x1862022294224256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767726 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767704.243113:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767704.243114:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767704.243116:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1e400. 02000000:00000001:3.0:1775767704.243119:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767704.243120:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767704.243122:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:3.0:1775767704.243124:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767704.243125:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029400. 02000000:00000001:3.0:1775767704.243128:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767704.243128:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767704.243130:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fe680. 00000100:00000001:3.0:1775767704.243132:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767704.243133:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767704.243136:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767704.243139:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fdc00 x1862022294224128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767726 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767704.243145:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767704.243146:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767704.243147:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1d000. 02000000:00000001:3.0:1775767704.243149:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767704.243150:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767704.243152:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:3.0:1775767704.243191:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767704.243193:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002bc00. 02000000:00000001:3.0:1775767704.243197:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767704.243199:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767704.243200:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fdc00. 00000100:00000001:3.0:1775767704.243203:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767704.243204:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767704.243206:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767704.243209:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fce00 x1862022294224000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767726 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767704.243216:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767704.243217:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767704.243218:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1fc00. 02000000:00000001:3.0:1775767704.243220:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767704.243221:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767704.243223:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:3.0:1775767704.243225:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767704.243226:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002b800. 02000000:00000001:3.0:1775767704.243228:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767704.243229:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767704.243230:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fce00. 00000100:00000001:3.0:1775767704.243232:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767704.243233:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767704.243235:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767704.243246:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767704.243248:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767704.243251:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767704.243253:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767705.265528:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767705.265534:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767705.265538:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767705.265542:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767705.265548:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767705.265555:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767705.265624:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767705.265627:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767705.265631:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767706.289356:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767706.289362:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775767706.289366:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767706.289371:0:36:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767706.289376:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767706.289388:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 4958 00000400:00000200:3.0:1775767706.289416:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767706.289423:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767706.289426:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767706.289430:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767706.289434:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767706.289437:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767706.289440:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767706.289445:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767706.289447:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:3.0:1775767706.289450:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767706.289453:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:3.0:1775767706.289456:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:3.0:1775767706.289465:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:3.0:1775767706.289469:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767706.289471:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767706.289473:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767706.289475:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767706.289479:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:3.0:1775767706.289483:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767706.289486:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767706.289489:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767706.289493:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767706.289496:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767706.289499:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775767706.289501:0:36:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767706.289508:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8a80. 00000020:00000040:3.0:1775767706.289512:0:36:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:3.0:1775767706.289516:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767706.289518:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767706.289520:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767706.289524:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767706.289528:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767706.289536:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2c00. 00000100:00000001:3.0:1775767706.289544:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767706.289554:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8a80 x1862022294224640/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767706.289566:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767706.289571:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767706.289576:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006c3265af] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8a80 x1862022294224640/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767706.289603:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767706.289607:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767706.289611:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775767706.289612:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767706.289615:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767706.289618:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767706.289622:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4958 00000400:00000200:3.0:1775767706.289627:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767706.289632:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767706.289635:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767706.289638:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767706.289643:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767706.289648:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767706.289653:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767706.289656:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767706.289657:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:3.0:1775767706.289660:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767706.289663:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:3.0:1775767706.289667:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:3.0:1775767706.289671:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:3.0:1775767706.289680:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767706.289687:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767706.289692:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767706.289697:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767706.289702:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00080000:3.0:1775767706.289710:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767706.289717:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767706.289721:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767706.289732:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767706.289749:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767706.289753:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767706.289763:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775767706.289768:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767706.289772:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767706.289779:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aa300. 00000020:00000040:3.0:1775767706.289785:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:3.0:1775767706.289789:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767706.289792:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767706.289795:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767706.289797:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767706.289800:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767706.289807:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0c00. 00000100:00000001:3.0:1775767706.289813:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767706.289822:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aa300 x1862022294224768/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767706.289840:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767706.289845:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767706.289855:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f33bdf15] to pc [ptlrpcd_rcv+-1] req@ffff9942099aa300 x1862022294224768/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767706.289869:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767706.289872:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767706.289876:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775767706.289878:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767706.289879:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767706.289882:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767706.289887:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4958 00000400:00000200:3.0:1775767706.289894:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767706.289901:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767706.289904:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767706.289912:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767706.289915:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767706.289919:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767706.289925:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767706.289929:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767706.289931:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:3.0:1775767706.289934:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767706.289938:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:3.0:1775767706.289941:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:3.0:1775767706.289947:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:3.0:1775767706.289955:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767706.289960:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:2.1:1775767706.290017:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767706.290021:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000100:00000040:3.0:1775767706.290223:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767706.290230:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767706.290236:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:3.0:1775767706.290243:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767706.290249:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767706.290254:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767706.290260:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767706.290266:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767706.290272:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767706.290277:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775767706.290282:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767706.290287:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767706.290294:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ab800. 00000020:00000040:3.0:1775767706.290298:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:3.0:1775767706.290307:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767706.290310:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767706.290314:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767706.290318:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767706.290322:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767706.290330:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2800. 00000100:00000001:3.0:1775767706.290337:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767706.290348:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ab800 x1862022294224896/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767706.290369:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767706.290375:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767706.290384:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000089df80c0] to pc [ptlrpcd_rcv+-1] req@ffff9942099ab800 x1862022294224896/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767706.290471:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767706.290476:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767706.290485:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775767706.290489:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767706.290493:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767706.290499:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767706.290506:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4958 00000400:00000200:3.0:1775767706.290520:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767706.290528:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767706.290532:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767706.290536:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767706.290540:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767706.290544:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767706.290550:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767706.290554:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767706.290556:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:3.0:1775767706.290560:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767706.290562:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:3.0:1775767706.290568:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:3.0:1775767706.290574:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:3.0:1775767706.290582:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767706.290587:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767706.290594:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767706.290599:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767706.290605:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:3.0:1775767706.290612:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767706.290619:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767706.290623:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767706.290629:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767706.290635:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767706.290639:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767706.290644:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:3.0:1775767706.290651:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767706.290656:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767706.290664:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aa680. 00000020:00000040:3.0:1775767706.290669:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:3.0:1775767706.290676:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767706.290678:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767706.290680:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767706.290682:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767706.290686:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767706.290693:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0000. 00000100:00000001:3.0:1775767706.290700:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767706.290709:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aa680 x1862022294225024/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:2.1:1775767706.290728:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:3.0:1775767706.290729:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000200:2.1:1775767706.290744:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000100:00000001:3.0:1775767706.290746:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767706.290755:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006cb1597d] to pc [ptlrpcd_rcv+-1] req@ffff9942099aa680 x1862022294225024/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767706.290774:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767706.290778:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767706.290786:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775767706.290790:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767706.290793:0:36:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767706.290800:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767706.290806:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4958 00000400:00000200:3.0:1775767706.290816:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767706.290825:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767706.290829:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767706.290833:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767706.290838:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767706.290842:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767706.290845:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767706.290850:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767706.290854:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:3.0:1775767706.290855:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767706.290857:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:3.0:1775767706.290859:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:3.0:1775767706.290863:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:3.0:1775767706.290866:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767706.290868:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767706.290871:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767706.290875:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767706.290878:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:3.0:1775767706.290883:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767706.290886:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767706.290890:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767706.290893:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767706.290896:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767706.290900:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767706.290903:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:3.0:1775767706.290906:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767706.290911:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767706.290916:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aad80. 00000020:00000040:3.0:1775767706.290925:0:36:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:3.0:1775767706.290931:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767706.290933:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767706.290936:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767706.290938:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767706.290942:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767706.290949:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2c00. 00000100:00000001:3.0:1775767706.290955:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767706.290965:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aad80 x1862022294225152/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767706.291009:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767706.291014:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767706.291024:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000045ccf08] to pc [ptlrpcd_rcv+-1] req@ffff9942099aad80 x1862022294225152/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767706.291049:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767706.291058:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (4968) 00000800:00000200:2.1:1775767706.291263:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767706.291265:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775767706.291644:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767706.291647:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775767706.291970:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767706.291973:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000100:00000001:3.0:1775767707.313265:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767707.313269:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767707.313273:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767707.313278:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767707.313281:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767707.313285:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767707.313288:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767707.313291:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767707.313294:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767707.633220:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299632445, at 4299632448 00000100:00000010:1.0:1775767707.633234:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d63c0. 00000100:00000001:3.0:1775767708.337216:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767708.337221:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767708.337224:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767708.337230:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767708.337234:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767708.337238:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767708.337244:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767708.337246:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767708.337249:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767709.361252:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767709.361259:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767709.361263:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767709.361272:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767709.361278:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767709.361282:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767709.361288:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767709.361291:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767709.361292:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767710.385224:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767710.385229:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767710.385232:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767710.385241:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767710.385248:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767710.385253:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767710.385265:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767710.385269:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767710.385272:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767711.409310:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767711.409324:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767711.409329:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767711.409330:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775767711.409333:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767711.409340:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767711.409344:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775767711.409349:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775767711.409352:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767711.409354:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775767711.409355:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767711.409361:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767711.409362:0:36:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767711.409369:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767711.409379:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 4963 00000400:00000200:3.0:1775767711.409396:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767711.409404:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767711.409408:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767711.409412:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767711.409416:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767711.409418:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767711.409421:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767711.409425:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767711.409427:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:3.0:1775767711.409431:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767711.409434:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:3.0:1775767711.409437:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:3.0:1775767711.409439:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:3.0:1775767711.409442:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767711.409445:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767711.409447:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767711.409448:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767711.409451:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:3.0:1775767711.409454:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767711.409459:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767711.409462:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767711.409467:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767711.409470:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767711.409473:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775767711.409476:0:36:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767711.409488:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8e00. 00000020:00000040:3.0:1775767711.409492:0:36:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:3.0:1775767711.409497:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767711.409499:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767711.409502:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767711.409505:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767711.409511:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767711.409520:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b1000. 00000100:00000001:3.0:1775767711.409528:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767711.409539:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8e00 x1862022294225280/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767711.409551:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767711.409565:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767711.409571:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000864a6866] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8e00 x1862022294225280/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767711.409595:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767711.409599:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767711.409602:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775767711.409605:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767711.409607:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767711.409612:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767711.409615:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4963 00000400:00000200:3.0:1775767711.409620:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767711.409627:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767711.409632:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767711.409643:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767711.409655:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767711.409665:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767711.409673:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767711.409678:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767711.409680:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:3.0:1775767711.409699:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767711.409703:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:3.0:1775767711.409708:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:3.0:1775767711.409718:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:3.0:1775767711.409728:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767711.409736:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767711.409742:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767711.409747:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767711.409757:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00080000:3.0:1775767711.409770:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767711.409778:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767711.409782:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767711.409792:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767711.409801:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767711.409805:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767711.409811:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775767711.409821:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767711.409828:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767711.409835:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ab480. 00000020:00000040:3.0:1775767711.409842:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:3.0:1775767711.409847:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767711.409849:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767711.409858:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767711.409861:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767711.409864:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767711.409875:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2000. 00000100:00000001:3.0:1775767711.409883:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767711.409914:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ab480 x1862022294225408/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767711.409949:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767711.409954:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767711.409966:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000067a0ba74] to pc [ptlrpcd_rcv+-1] req@ffff9942099ab480 x1862022294225408/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.1:1775767711.409973:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767711.409982:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000100:00000001:3.0:1775767711.410009:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767711.410014:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767711.410026:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775767711.410030:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767711.410035:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767711.410041:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767711.410051:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4963 00000400:00000200:3.0:1775767711.410068:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767711.410081:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767711.410095:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767711.410104:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767711.410110:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767711.410114:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767711.410117:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767711.410122:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767711.410125:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:3.0:1775767711.410131:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767711.410135:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:3.0:1775767711.410140:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:3.0:1775767711.410145:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:3.0:1775767711.410150:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767711.410153:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767711.410191:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767711.410200:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767711.410203:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:3.0:1775767711.410211:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767711.410216:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767711.410220:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767711.410225:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767711.410229:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767711.410239:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767711.410244:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775767711.410255:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767711.410259:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767711.410268:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a9180. 00000020:00000040:3.0:1775767711.410273:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:3.0:1775767711.410282:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767711.410286:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767711.410292:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767711.410295:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767711.410298:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767711.410309:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0400. 00000100:00000001:3.0:1775767711.410319:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767711.410338:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a9180 x1862022294225536/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767711.410389:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767711.410393:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767711.410403:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bdc2586] to pc [ptlrpcd_rcv+-1] req@ffff9942099a9180 x1862022294225536/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767711.410439:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767711.410446:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767711.410450:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775767711.410452:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767711.410455:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767711.410459:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767711.410467:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4963 00000400:00000200:3.0:1775767711.410479:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767711.410486:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767711.410488:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767711.410495:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767711.410506:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767711.410510:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767711.410515:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767711.410522:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767711.410525:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:3.0:1775767711.410528:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767711.410530:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:3.0:1775767711.410536:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:3.0:1775767711.410547:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:3.0:1775767711.410559:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767711.410564:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767711.410574:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767711.410578:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767711.410583:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:3.0:1775767711.410594:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767711.410603:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767711.410611:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767711.410620:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767711.410627:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767711.410636:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767711.410641:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:3.0:1775767711.410647:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767711.410657:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767711.410665:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8700. 00000020:00000040:3.0:1775767711.410673:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:3.0:1775767711.410679:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767711.410681:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767711.410703:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767711.410706:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767711.410713:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767711.410720:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b3400. 00000100:00000001:3.0:1775767711.410726:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767711.410737:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8700 x1862022294225664/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767711.410754:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767711.410764:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767711.410777:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000faa25e3b] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8700 x1862022294225664/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767711.410800:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767711.410814:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767711.410823:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775767711.410826:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767711.410831:0:36:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767711.410838:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767711.410848:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4963 00000400:00000200:3.0:1775767711.410858:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767711.410863:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767711.410869:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767711.410876:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767711.410885:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767711.410888:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767711.410895:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767711.410900:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767711.410905:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:3.0:1775767711.410910:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767711.410915:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:3.0:1775767711.410921:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:3.0:1775767711.410932:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:3.0:1775767711.410945:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767711.410951:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767711.410961:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767711.410968:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767711.410976:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:3.0:1775767711.410984:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767711.410993:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767711.410995:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767711.411003:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767711.411009:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767711.411014:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767711.411020:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:3.0:1775767711.411029:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767711.411035:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767711.411042:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8380. 00000020:00000040:3.0:1775767711.411048:0:36:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:3.0:1775767711.411055:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767711.411058:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767711.411063:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767711.411068:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767711.411071:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767711.411082:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b1000. 00000100:00000001:3.0:1775767711.411089:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767711.411102:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8380 x1862022294225792/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767711.411115:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767711.411119:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767711.411128:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055f5621a] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8380 x1862022294225792/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767711.411141:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775767711.411151:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000040:3.0:1775767711.411152:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (4973) 00000800:00000200:2.1:1775767711.411177:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775767711.411381:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767711.411382:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775767711.411651:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767711.411654:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775767711.411849:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767711.411851:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000100:00000001:3.0:1775767712.433300:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767712.433301:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767712.433306:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767712.433308:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767712.433309:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767712.433311:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767712.433321:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775767712.433322:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767712.433332:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767712.433332:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767712.433341:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767712.433343:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767712.433349:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767712.433352:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767712.433355:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767712.433366:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767712.433371:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767712.433375:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767712.433473:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767712.433476:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767712.433478:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767712.433481:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767712.433483:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767712.433486:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767712.433491:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767712.433494:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767712.433498:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767712.753246:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299637566, at 4299637568 00000100:00000010:1.0:1775767712.753259:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d6e40. 00000100:00000001:3.0:1775767713.457288:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767713.457298:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767713.457301:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767713.457310:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:3.0:1775767713.457313:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767713.457321:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767713.457322:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767713.457325:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767713.457331:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767713.457331:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767713.457341:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767713.457342:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767713.457347:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767713.457354:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767713.457355:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767713.457368:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767713.457374:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767713.457378:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767714.481294:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767714.481313:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767714.481318:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767714.481321:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767714.481331:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:3.0:1775767714.481331:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767714.481336:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767714.481338:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1775767714.481345:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767714.481347:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767714.481357:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767714.481358:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767714.481368:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767714.481368:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767714.481374:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767714.481374:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767714.481376:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767714.481380:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767715.505222:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767715.505227:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767715.505229:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767715.505241:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767715.505249:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767715.505253:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767715.505254:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767715.505261:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767715.505264:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767715.505265:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767715.505271:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767715.505271:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767715.505276:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767715.505280:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767715.505283:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767715.505293:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767715.505297:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767715.505301:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767716.529270:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767716.529275:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767716.529277:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767716.529293:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767716.529296:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767716.529303:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767716.529306:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767716.529309:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767716.529312:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:3.0:1775767716.529362:0:36:0:(rw.c:1481:ll_writepages()) Process entered 00000080:00000001:3.0:1775767716.529367:0:36:0:(file.c:5250:cl_sync_file_range()) Process entered 00000020:00000001:3.0:1775767716.529371:0:36:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00080000:1.0:1775767716.529375:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767716.529380:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000020:00000010:3.0:1775767716.529383:0:36:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff99420693e000. 00000100:00000001:1.0:1775767716.529387:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775767716.529397:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000020:00000010:3.0:1775767716.529400:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcf000. 00000100:00000001:1.0:1775767716.529411:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000020:00000010:3.0:1775767716.529414:0:36:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210dcf400. 00000100:00080000:1.0:1775767716.529420:0:13471:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 4968 00000020:00000010:3.0:1775767716.529424:0:36:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207981960. 00000008:00000010:3.0:1775767716.529434:0:36:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994208b50000. 00000400:00000200:1.0:1775767716.529441:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000010:3.0:1775767716.529446:0:36:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994218000300. 00000400:00000200:1.0:1775767716.529451:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00008000:00000010:3.0:1775767716.529454:0:36:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994207e08e20. 00000100:00080000:1.0:1775767716.529455:0:13471:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775767716.529467:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000080:00000010:3.0:1775767716.529468:0:36:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99420541b1f8. 00000100:00000001:1.0:1775767716.529481:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000080:00000010:3.0:1775767716.529483:0:36:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077c0248. 00000100:00000040:1.0:1775767716.529489:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000010:3.0:1775767716.529492:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dce400. 00000100:00000001:1.0:1775767716.529493:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775767716.529503:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000008:00000010:3.0:1775767716.529504:0:36:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942086f8420. 00000020:00000040:1.0:1775767716.529505:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:1.0:1775767716.529509:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00020000:00000010:3.0:1775767716.529510:0:36:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994207e8e400. 00000020:00000040:1.0:1775767716.529511:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00008000:00000010:3.0:1775767716.529516:0:36:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994209270a90. 00000020:00000001:1.0:1775767716.529517:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:1.0:1775767716.529529:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000080:00000010:3.0:1775767716.529531:0:36:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942090e33c0. 00000020:00000001:3.0:1775767716.529542:0:36:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107590086664 : -112966119464952 : ffff99420693e008) 00000100:00000040:1.0:1775767716.529543:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00001000:3.0:1775767716.529553:0:36:0:(cl_object.c:831:cl_env_get()) 1@ffff99420693e000 00000100:00000001:1.0:1775767716.529554:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767716.529562:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767716.529564:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767716.529564:0:36:0:(cl_object.c:151:cl_object_top()) ffff99420791a5f8 -> ffff99420791a5f8 00000020:00000001:3.0:1775767716.529571:0:36:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000040:1.0:1775767716.529572:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000080:00000001:3.0:1775767716.529575:0:36:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000100:00080000:1.0:1775767716.529581:0:13471:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000080:00200000:3.0:1775767716.529582:0:36:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x10176:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000100:00000001:1.0:1775767716.529591:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767716.529592:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:3.0:1775767716.529598:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 02000000:00000001:1.0:1775767716.529599:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000080:00000001:3.0:1775767716.529602:0:36:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767716.529604:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767716.529608:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:3.0:1775767716.529609:0:36:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x10176:0x0]io ffff9942077c02a8 type 6 ignore/verify layout 1/0 00000100:00000001:1.0:1775767716.529610:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00020000:00000002:3.0:1775767716.529616:0:36:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207dfb6c8) owner 0000000000000000/ffff994201348000 00000100:00000001:1.0:1775767716.529617:0:13471:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767716.529623:0:36:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:3.0:1775767716.529625:0:36:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:3.0:1775767716.529628:0:36:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00020000:00000001:3.0:1775767716.529629:0:36:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767716.529632:0:36:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:3.0:1775767716.529651:0:36:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00000100:00000010:1.0:1775767716.529654:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207889880. 00020000:00000001:3.0:1775767716.529654:0:36:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1775767716.529663:0:13471:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:1.0:1775767716.529678:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775767716.529679:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767716.529679:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:3.0:1775767716.529684:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00000001:1.0:1775767716.529686:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00020000:00000001:3.0:1775767716.529688:0:36:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 02000000:00000001:1.0:1775767716.529689:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00020000:00000002:3.0:1775767716.529694:0:36:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207dfb6c8) owner 0000000000000000/ffff994201348000 02000000:00000001:1.0:1775767716.529696:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767716.529707:0:36:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775767716.529713:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543f800. 00000020:00000001:3.0:1775767716.529713:0:36:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:3.0:1775767716.529720:0:36:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000100:00000001:1.0:1775767716.529723:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767716.529724:0:36:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00020000:00200000:3.0:1775767716.529730:0:36:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00100000:1.0:1775767716.529737:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207889880 x1862022294225920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00020000:00000002:3.0:1775767716.529738:0:36:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:3.0:1775767716.529754:0:36:0:(lov_io.c:155:lov_sub_get()) Process entered 00020000:00000001:3.0:1775767716.529759:0:36:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000020:00000001:3.0:1775767716.529763:0:36:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00000001:1.0:1775767716.529764:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000020:00000010:3.0:1775767716.529776:0:36:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff99420693ecf0. 00000100:00000001:1.0:1775767716.529777:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1775767716.529790:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dce200. 00000100:00000040:1.0:1775767716.529794:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f11856b8] to pc [ptlrpcd_rcv+-1] req@ffff994207889880 x1862022294225920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1775767716.529796:0:36:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210dcea00. 00000020:00000010:3.0:1775767716.529809:0:36:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942079810c0. 00000008:00000010:3.0:1775767716.529813:0:36:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994208b51650. 00000100:00000001:1.0:1775767716.529814:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767716.529818:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767716.529820:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00020000:00000010:3.0:1775767716.529821:0:36:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994218000000. 00000100:00000001:1.0:1775767716.529823:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775767716.529827:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775767716.529840:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775767716.529844:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4968 00008000:00000010:3.0:1775767716.529844:0:36:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994207e0ade8. 00000080:00000010:3.0:1775767716.529850:0:36:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99420541b0a8. 00000400:00000200:1.0:1775767716.529859:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000080:00000010:3.0:1775767716.529861:0:36:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077c1da8. 00000020:00000010:3.0:1775767716.529865:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dce000. 00000400:00000200:1.0:1775767716.529867:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000008:00000010:3.0:1775767716.529874:0:36:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942086f8160. 00000100:00080000:1.0:1775767716.529876:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00020000:00000010:3.0:1775767716.529884:0:36:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994207e8ed00. 00000100:00000040:1.0:1775767716.529888:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767716.529893:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00008000:00000010:3.0:1775767716.529893:0:36:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994209270ef0. 00000080:00000010:3.0:1775767716.529900:0:36:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942090e3960. 00000100:00000040:1.0:1775767716.529901:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767716.529910:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767716.529910:0:36:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107590089976 : -112966119461640 : ffff99420693ecf8) 00000020:00000001:1.0:1775767716.529924:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00001000:3.0:1775767716.529924:0:36:0:(cl_object.c:831:cl_env_get()) 1@ffff99420693ecf0 00000020:00000040:1.0:1775767716.529931:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:3.0:1775767716.529934:0:36:0:(cl_object.c:151:cl_object_top()) ffff99420791ae90 -> ffff99420791a5f8 00000020:00000001:1.0:1775767716.529935:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775767716.529941:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:3.0:1775767716.529941:0:36:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:3.0:1775767716.529948:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:1.0:1775767716.529949:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:3.0:1775767716.529955:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:1.0:1775767716.529956:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:3.0:1775767716.529961:0:36:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767716.529972:0:36:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767716.529973:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00020000:00000001:3.0:1775767716.529984:0:36:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107612435560 : -112966097116056 : ffff994207e8e468) 00000100:00000001:1.0:1775767716.529985:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767716.529993:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000001:3.0:1775767716.529996:0:36:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000100:00000001:1.0:1775767716.529998:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767716.530001:0:36:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1775767716.530006:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00020000:00000001:3.0:1775767716.530008:0:36:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767716.530015:0:36:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767716.530016:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000020:00000001:3.0:1775767716.530023:0:36:0:(cl_io.c:314:cl_io_lock()) Process entered 00020000:00000001:3.0:1775767716.530028:0:36:0:(lov_io.c:1108:lov_io_lock()) Process entered 00000100:00000001:1.0:1775767716.530029:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767716.530032:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 02000000:00000001:1.0:1775767716.530038:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000020:00000001:3.0:1775767716.530038:0:36:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:3.0:1775767716.530043:0:36:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:3.0:1775767716.530046:0:36:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767716.530048:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1775767716.530048:0:36:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767716.530056:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767716.530059:0:36:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767716.530060:0:36:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:3.0:1775767716.530060:0:36:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767716.530062:0:36:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767716.530069:0:36:0:(cl_io.c:528:cl_io_start()) Process entered 02000000:00000001:1.0:1775767716.530070:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767716.530072:0:36:0:(lov_io.c:1114:lov_io_start()) Process entered 00020000:00000001:3.0:1775767716.530075:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000001:1.0:1775767716.530076:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000020:00000001:3.0:1775767716.530079:0:36:0:(cl_io.c:528:cl_io_start()) Process entered 00000008:00000001:3.0:1775767716.530082:0:36:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000008:00100000:1.0:1775767716.530083:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775767716.530085:0:36:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000008:00000001:1.0:1775767716.530092:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1775767716.530094:0:36:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000100:00000001:1.0:1775767716.530096:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000008:00000020:3.0:1775767716.530099:0:36:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:3.0:1775767716.530103:0:36:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:1.0:1775767716.530106:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207888e00. 00000008:00000001:3.0:1775767716.530110:0:36:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000020:00000040:1.0:1775767716.530111:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:1.0:1775767716.530128:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775767716.530132:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1775767716.530132:0:36:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000100:00000001:1.0:1775767716.530139:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000008:00000001:3.0:1775767716.530139:0:36:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107754940544 : -112965954611072 : ffff994210675880) 02000000:00000001:1.0:1775767716.530144:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775767716.530148:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000020:3.0:1775767716.530150:0:36:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994210675880 ready 0|-|- wr 1|-|+ rd 0|- 10 in flight 02000000:00000010:1.0:1775767716.530153:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543d000. 00000008:00000001:3.0:1775767716.530157:0:36:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:3.0:1775767716.530160:0:36:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:3.0:1775767716.530163:0:36:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1775767716.530167:0:36:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000008:00000020:3.0:1775767716.530170:0:36:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994218232fc0 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000008:00000001:3.0:1775767716.530174:0:36:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1775767716.530177:0:36:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767716.530187:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767716.530201:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207888e00 x1862022294226048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775767716.530211:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000020:00000001:3.0:1775767716.530212:0:36:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767716.530215:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767716.530220:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767716.530224:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001812f966] to pc [ptlrpcd_rcv+-1] req@ffff994207888e00 x1862022294226048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:3.0:1775767716.530227:0:36:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767716.530237:0:36:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767716.530247:0:36:0:(cl_io.c:555:cl_io_end()) Process entered 00020000:00000001:3.0:1775767716.530255:0:36:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00000100:00000001:1.0:1775767716.530256:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767716.530262:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00020000:00000001:3.0:1775767716.530262:0:36:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000020:00000001:3.0:1775767716.530268:0:36:0:(cl_io.c:555:cl_io_end()) Process entered 00000100:00080000:1.0:1775767716.530270:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000020:00000001:3.0:1775767716.530275:0:36:0:(cl_io.c:563:cl_io_end()) Process leaving 00000100:00000001:1.0:1775767716.530276:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00020000:00000001:3.0:1775767716.530278:0:36:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767716.530280:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775767716.530296:0:13471:0:(import.c:484:import_select_connection()) Process entered 00020000:00000001:3.0:1775767716.530303:0:36:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000100:00080000:1.0:1775767716.530308:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4968 00000020:00000001:3.0:1775767716.530308:0:36:0:(cl_io.c:563:cl_io_end()) Process leaving 00000020:00000001:3.0:1775767716.530315:0:36:0:(cl_io.c:352:cl_io_unlock()) Process entered 00020000:00000001:3.0:1775767716.530318:0:36:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00020000:00000001:3.0:1775767716.530323:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000400:00000200:1.0:1775767716.530324:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:3.0:1775767716.530328:0:36:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000020:00000001:3.0:1775767716.530333:0:36:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000400:00000200:1.0:1775767716.530338:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00020000:00000001:3.0:1775767716.530338:0:36:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767716.530346:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767716.530347:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00020000:00000001:3.0:1775767716.530357:0:36:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000100:00000040:1.0:1775767716.530362:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000001:3.0:1775767716.530362:0:36:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:3.0:1775767716.530364:0:36:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000020:00000001:3.0:1775767716.530368:0:36:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000001:3.0:1775767716.530372:0:36:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000100:00000001:1.0:1775767716.530374:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00020000:00000001:3.0:1775767716.530376:0:36:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00020000:00000001:3.0:1775767716.530380:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000040:1.0:1775767716.530382:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000001:3.0:1775767716.530384:0:36:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000001:3.0:1775767716.530389:0:36:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00020000:00000001:3.0:1775767716.530393:0:36:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767716.530395:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00020000:00000001:3.0:1775767716.530397:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767716.530404:0:36:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000020:00000001:1.0:1775767716.530405:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000001:3.0:1775767716.530406:0:36:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000020:00000001:3.0:1775767716.530411:0:36:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1775767716.530413:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:3.0:1775767716.530418:0:36:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:1.0:1775767716.530419:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00020000:00000001:3.0:1775767716.530422:0:36:0:(lov_io.c:675:lov_io_fini()) Process entered 00000020:00000040:1.0:1775767716.530424:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00020000:00000001:3.0:1775767716.530425:0:36:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:3.0:1775767716.530429:0:36:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:3.0:1775767716.530436:0:36:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00000001:1.0:1775767716.530437:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00001000:3.0:1775767716.530439:0:36:0:(cl_object.c:925:cl_env_put()) 1@ffff99420693ecf0 00000020:00000001:1.0:1775767716.530448:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000010:3.0:1775767716.530449:0:36:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210dcea00. 00000100:00000040:1.0:1775767716.530463:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000010:3.0:1775767716.530465:0:36:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942079810c0. 00000800:00000200:2.1:1775767716.530469:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:1.0:1775767716.530471:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:2.1:1775767716.530473:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000008:00000010:3.0:1775767716.530474:0:36:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994208b51650. 00000100:00000040:1.0:1775767716.530477:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767716.530488:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775767716.530491:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00020000:00000010:3.0:1775767716.530491:0:36:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994218000000. 00008000:00000010:3.0:1775767716.530500:0:36:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994207e0ade8. 00000100:00080000:1.0:1775767716.530504:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000080:00000010:3.0:1775767716.530510:0:36:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99420541b0a8. 00000100:00000001:1.0:1775767716.530514:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767716.530517:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000080:00000010:3.0:1775767716.530518:0:36:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077c1da8. 02000000:00000001:1.0:1775767716.530522:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1775767716.530532:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dce200. 02000000:00000001:1.0:1775767716.530533:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767716.530545:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00000010:3.0:1775767716.530545:0:36:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942086f8160. 00000008:00100000:1.0:1775767716.530552:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00020000:00000010:3.0:1775767716.530554:0:36:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994207e8ed00. 00000008:00000001:1.0:1775767716.530565:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:3.0:1775767716.530566:0:36:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994209270ef0. 00000100:00000001:1.0:1775767716.530576:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:3.0:1775767716.530578:0:36:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942090e3960. 00000100:00000010:1.0:1775767716.530580:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff99420788b100. 00000020:00000010:3.0:1775767716.530587:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dce000. 00000020:00000040:1.0:1775767716.530588:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000010:3.0:1775767716.530597:0:36:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff99420693ecf0. 02000000:00000001:1.0:1775767716.530598:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775767716.530606:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767716.530606:0:36:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00000100:00000001:1.0:1775767716.530612:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00020000:00000001:3.0:1775767716.530612:0:36:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:3.0:1775767716.530619:0:36:0:(vvp_io.c:280:vvp_io_fini()) Process entered 02000000:00000001:1.0:1775767716.530620:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775767716.530627:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000080:00200000:3.0:1775767716.530627:0:36:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x10176:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 02000000:00000010:1.0:1775767716.530653:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543e400. 00000100:00000001:1.0:1775767716.530662:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:3.0:1775767716.530666:0:36:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000100:00100000:1.0:1775767716.530673:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff99420788b100 x1862022294226176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000001:3.0:1775767716.530673:0:36:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:3.0:1775767716.530680:0:36:0:(cl_object.c:925:cl_env_put()) 1@ffff99420693e000 00000020:00000010:3.0:1775767716.530690:0:36:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210dcf400. 00000100:00000001:1.0:1775767716.530692:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775767716.530699:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1775767716.530702:0:36:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207981960. 00000100:00000040:1.0:1775767716.530704:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e26cf977] to pc [ptlrpcd_rcv+-1] req@ffff99420788b100 x1862022294226176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000010:3.0:1775767716.530710:0:36:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994208b50000. 00020000:00000010:3.0:1775767716.530720:0:36:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994218000300. 00008000:00000010:3.0:1775767716.530731:0:36:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994207e08e20. 00000100:00000001:1.0:1775767716.530732:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767716.530738:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000080:00000010:3.0:1775767716.530739:0:36:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99420541b1f8. 00000080:00000010:3.0:1775767716.530745:0:36:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077c0248. 00000020:00000010:3.0:1775767716.530757:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcf000. 00000100:00080000:1.0:1775767716.530758:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775767716.530767:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775767716.530771:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000008:00000010:3.0:1775767716.530771:0:36:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942086f8420. 00020000:00000010:3.0:1775767716.530784:0:36:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994207e8e400. 00000100:00000001:1.0:1775767716.530785:0:13471:0:(import.c:484:import_select_connection()) Process entered 00008000:00000010:3.0:1775767716.530795:0:36:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994209270a90. 00000100:00080000:1.0:1775767716.530796:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4968 00000080:00000010:3.0:1775767716.530805:0:36:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942090e33c0. 00000400:00000200:1.0:1775767716.530813:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000010:3.0:1775767716.530813:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dce400. 00000020:00000010:3.0:1775767716.530822:0:36:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff99420693e000. 00000400:00000200:1.0:1775767716.530827:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000080:00000001:3.0:1775767716.530831:0:36:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1775767716.530836:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000080:00000001:3.0:1775767716.530843:0:36:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767716.530849:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767716.530855:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767716.530858:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767716.530862:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775767716.530866:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775767716.530868:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:1.0:1775767716.530870:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775767716.530872:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:1.0:1775767716.530874:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:1.0:1775767716.530879:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:1.0:1775767716.530884:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767716.530888:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767716.530892:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767716.530895:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775767716.530898:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:1.0:1775767716.530902:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775767716.530907:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767716.530910:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775767716.530914:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767716.530917:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767716.530919:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775767716.530922:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:1.0:1775767716.530927:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767716.530931:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775767716.530937:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207889f80. 00000020:00000040:1.0:1775767716.530941:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:1.0:1775767716.530945:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775767716.530947:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767716.530951:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775767716.530953:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775767716.530955:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775767716.530959:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543d400. 00000100:00000001:1.0:1775767716.530964:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767716.530971:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207889f80 x1862022294226304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775767716.530993:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775767716.530996:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767716.531002:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ed2274f4] to pc [ptlrpcd_rcv+-1] req@ffff994207889f80 x1862022294226304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767716.531013:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767716.531016:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767716.531020:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775767716.531023:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775767716.531026:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775767716.531030:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775767716.531035:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4968 00000400:00000200:1.0:1775767716.531042:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767716.531047:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775767716.531051:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775767716.531058:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767716.531059:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767716.531063:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767716.531067:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775767716.531070:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775767716.531072:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:1.0:1775767716.531075:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775767716.531077:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:1.0:1775767716.531081:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:1.0:1775767716.531085:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:1.0:1775767716.531092:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767716.531096:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767716.531100:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767716.531103:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775767716.531105:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:1.0:1775767716.531109:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775767716.531113:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767716.531115:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775767716.531117:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767716.531120:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767716.531122:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775767716.531124:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:1.0:1775767716.531127:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767716.531129:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775767716.531133:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207888700. 00000020:00000040:1.0:1775767716.531136:0:13471:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:1.0:1775767716.531139:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775767716.531140:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767716.531144:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775767716.531145:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775767716.531146:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775767716.531151:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543dc00. 00000100:00000001:1.0:1775767716.531174:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767716.531179:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207888700 x1862022294226432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775767716.531196:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775767716.531199:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767716.531203:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009f2ba301] to pc [ptlrpcd_rcv+-1] req@ffff994207888700 x1862022294226432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767716.531214:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767716.531218:0:13471:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (4979) 00000800:00000200:2.1:1775767716.531331:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767716.531334:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1775767716.531549:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767716.531551:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1775767716.531789:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767716.531791:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1775767716.532070:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767716.532072:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000100:00000001:1.0:1775767717.553272:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767717.553279:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767717.553283:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767717.553289:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767717.553294:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767717.553299:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767717.553315:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767717.553318:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767717.553322:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767717.873283:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299642686, at 4299642688 00000100:00000010:1.0:1775767717.873304:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d6f00. 00000100:00000001:1.0:1775767718.577293:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767718.577300:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767718.577305:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767718.577313:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767718.577320:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767718.577343:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767718.577350:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767718.577354:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767718.577366:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767719.601257:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767719.601264:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767719.601268:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767719.601269:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:1.0:1775767719.601274:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767719.601275:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767719.601278:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767719.601279:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1775767719.601284:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767719.601285:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767719.601290:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767719.601291:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767719.601294:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767719.601295:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767719.601299:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767719.601300:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767719.601304:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767719.601309:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767720.625252:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767720.625252:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767720.625259:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767720.625260:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767720.625263:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767720.625264:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767720.625270:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:1.0:1775767720.625271:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767720.625277:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767720.625278:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767720.625282:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767720.625283:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767720.625288:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767720.625291:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767720.625293:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767720.625295:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767720.625298:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767720.625300:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767721.649269:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767721.649275:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:3.0:1775767721.649278:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775767721.649279:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767721.649286:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00080000:3.0:1775767721.649288:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775767721.649293:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767721.649294:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775767721.649298:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775767721.649301:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775767721.649303:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767721.649307:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767721.649309:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775767721.649311:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767721.649319:0:13471:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 4974 00000400:00000200:3.0:1775767721.649331:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767721.649332:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767721.649337:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767721.649340:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767721.649343:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00100000:1.0:1775767721.649344:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767721.649350:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1775767721.649351:0:13471:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1775767721.649354:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767721.649358:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767721.649359:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767721.649364:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767721.649365:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775767721.649369:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767721.649371:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767721.649376:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767721.649382:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767721.649386:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:3.0:1775767721.649389:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767721.649392:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:3.0:1775767721.649395:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:3.0:1775767721.649399:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:3.0:1775767721.649407:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767721.649412:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767721.649417:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767721.649421:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767721.649427:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:3.0:1775767721.649435:0:13471:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767721.649441:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767721.649448:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767721.649454:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767721.649460:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767721.649464:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775767721.649469:0:13471:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767721.649480:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8380. 00000020:00000040:3.0:1775767721.649486:0:13471:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:3.0:1775767721.649493:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767721.649497:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767721.649503:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767721.649507:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767721.649513:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767721.649524:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2c00. 00000100:00000001:3.0:1775767721.649533:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767721.649545:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8380 x1862022294226560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767721.649568:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767721.649575:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767721.649584:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055f5621a] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8380 x1862022294226560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767721.649629:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767721.649635:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767721.649644:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775767721.649648:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767721.649654:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767721.649663:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767721.649670:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4974 00000400:00000200:3.0:1775767721.649682:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767721.649690:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767721.649696:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767721.649705:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767721.649713:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767721.649720:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767721.649726:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767721.649733:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767721.649737:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:3.0:1775767721.649742:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767721.649746:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:3.0:1775767721.649753:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:3.0:1775767721.649758:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:3.0:1775767721.649766:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767721.649772:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767721.649777:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767721.649783:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767721.649790:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00080000:3.0:1775767721.649798:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767721.649807:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767721.649814:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767721.649822:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767721.649828:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767721.649834:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767721.649839:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775767721.649845:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767721.649851:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767721.649860:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8700. 00000020:00000040:3.0:1775767721.649867:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:3.0:1775767721.649875:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767721.649878:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767721.649883:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767721.649887:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767721.649892:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767721.649902:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0000. 00000100:00000001:3.0:1775767721.649909:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767721.649919:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8700 x1862022294226688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767721.649945:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767721.649951:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767721.649961:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000faa25e3b] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8700 x1862022294226688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.1:1775767721.649973:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767721.649980:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000100:00000001:3.0:1775767721.649993:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767721.650001:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767721.650011:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775767721.650015:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767721.650019:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767721.650024:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767721.650031:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4974 00000400:00000200:3.0:1775767721.650043:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767721.650053:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767721.650059:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767721.650067:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767721.650075:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767721.650081:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767721.650088:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767721.650093:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767721.650097:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:3.0:1775767721.650103:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767721.650106:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:3.0:1775767721.650112:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:3.0:1775767721.650119:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:3.0:1775767721.650125:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767721.650130:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767721.650137:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767721.650142:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767721.650147:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:3.0:1775767721.650182:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767721.650188:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767721.650194:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767721.650200:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767721.650207:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767721.650211:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767721.650215:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775767721.650223:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767721.650229:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767721.650238:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a9180. 00000020:00000040:3.0:1775767721.650245:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:3.0:1775767721.650253:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767721.650257:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767721.650263:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767721.650267:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767721.650271:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767721.650280:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2800. 00000100:00000001:3.0:1775767721.650287:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767721.650297:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a9180 x1862022294226816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767721.650330:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767721.650335:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767721.650347:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bdc2586] to pc [ptlrpcd_rcv+-1] req@ffff9942099a9180 x1862022294226816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767721.650383:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767721.650390:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767721.650400:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775767721.650405:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767721.650410:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767721.650417:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767721.650425:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4974 00000400:00000200:3.0:1775767721.650437:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767721.650447:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767721.650453:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767721.650463:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767721.650470:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767721.650477:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767721.650484:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767721.650491:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767721.650495:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:3.0:1775767721.650500:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767721.650506:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:3.0:1775767721.650512:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:3.0:1775767721.650518:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:3.0:1775767721.650528:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767721.650535:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767721.650541:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767721.650547:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767721.650554:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:3.0:1775767721.650563:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767721.650572:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767721.650577:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767721.650584:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767721.650603:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767721.650610:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767721.650616:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:3.0:1775767721.650623:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767721.650629:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767721.650638:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ab480. 00000020:00000040:3.0:1775767721.650643:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:3.0:1775767721.650648:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767721.650652:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767721.650657:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767721.650661:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767721.650666:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767721.650675:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0c00. 00000100:00000001:3.0:1775767721.650683:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767721.650695:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ab480 x1862022294226944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767721.650718:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767721.650724:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767721.650734:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000067a0ba74] to pc [ptlrpcd_rcv+-1] req@ffff9942099ab480 x1862022294226944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767721.650765:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767721.650772:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767721.650783:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775767721.650788:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767721.650791:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767721.650798:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767721.650804:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4974 00000400:00000200:3.0:1775767721.650817:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767721.650825:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767721.650832:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000800:00000200:2.1:1775767721.650837:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000040:3.0:1775767721.650841:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000800:00000200:2.1:1775767721.650841:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000100:00000001:3.0:1775767721.650848:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767721.650855:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767721.650862:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767721.650869:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767721.650873:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:3.0:1775767721.650878:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767721.650883:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:3.0:1775767721.650889:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:3.0:1775767721.650897:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:3.0:1775767721.650907:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767721.650913:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767721.650920:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767721.650928:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767721.650934:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:3.0:1775767721.650941:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767721.650948:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767721.650954:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767721.650960:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767721.650966:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767721.650972:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767721.650976:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:3.0:1775767721.650981:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767721.650987:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767721.650996:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8e00. 00000020:00000040:3.0:1775767721.651003:0:13471:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:3.0:1775767721.651011:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767721.651015:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767721.651021:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767721.651026:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767721.651030:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767721.651039:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2c00. 00000100:00000001:3.0:1775767721.651048:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767721.651060:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8e00 x1862022294227072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767721.651088:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767721.651094:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767721.651108:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000864a6866] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8e00 x1862022294227072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767721.651144:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767721.651151:0:13471:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (4984) 00000800:00000200:2.1:1775767721.651424:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767721.651426:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775767721.651687:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767721.651689:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775767721.651867:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767721.651869:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000100:00000001:1.0:1775767722.673284:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767722.673291:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767722.673294:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767722.673299:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767722.673305:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767722.673309:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767722.673309:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767722.673316:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767722.673319:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767722.673320:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767722.673323:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767722.673324:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767722.673328:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767722.673330:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767722.673335:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767722.673338:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767722.673340:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767722.673343:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767722.993348:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299647805, at 4299647808 00000100:00000010:1.0:1775767722.993355:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d6300. 00000100:00000001:1.0:1775767723.697273:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767723.697277:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767723.697282:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767723.697283:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767723.697285:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767723.697286:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767723.697294:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:1.0:1775767723.697295:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767723.697302:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767723.697302:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767723.697309:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767723.697309:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767723.697316:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767723.697319:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767723.697322:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767723.697325:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767723.697328:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767723.697332:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767724.721308:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767724.721308:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767724.721319:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767724.721319:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767724.721323:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767724.721323:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767724.721334:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775767724.721334:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767724.721341:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767724.721341:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767724.721347:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767724.721353:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767724.721357:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767724.721361:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767724.721365:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767724.721366:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767724.721373:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767724.721378:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767725.745243:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767725.745243:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767725.745249:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767725.745250:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767725.745253:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767725.745253:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767725.745261:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775767725.745261:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767725.745267:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767725.745267:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767725.745272:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767725.745272:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767725.745283:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767725.745285:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767725.745288:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767725.745288:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767725.745293:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767725.745297:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767726.769254:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767726.769260:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767726.769262:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767726.769263:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775767726.769268:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00080000:3.0:1775767726.769270:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775767726.769273:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767726.769274:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775767726.769277:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775767726.769278:0:36:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775767726.769282:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767726.769283:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775767726.769285:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767726.769289:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767726.769292:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 4979 00000400:00000200:3.0:1775767726.769302:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767726.769309:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767726.769314:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767726.769320:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767726.769324:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767726.769327:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767726.769329:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767726.769332:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767726.769338:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:3.0:1775767726.769340:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767726.769343:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:3.0:1775767726.769345:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:3.0:1775767726.769347:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:3.0:1775767726.769350:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767726.769352:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767726.769355:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767726.769358:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767726.769361:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:3.0:1775767726.769365:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767726.769369:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767726.769373:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767726.769377:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767726.769382:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767726.769387:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775767726.769392:0:36:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767726.769402:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aad80. 00000020:00000040:3.0:1775767726.769408:0:36:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:3.0:1775767726.769413:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767726.769416:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767726.769418:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767726.769421:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767726.769425:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767726.769432:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b1000. 00000100:00000001:3.0:1775767726.769438:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767726.769447:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aad80 x1862022294227200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767726.769456:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767726.769460:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767726.769464:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000045ccf08] to pc [ptlrpcd_rcv+-1] req@ffff9942099aad80 x1862022294227200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767726.769476:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767726.769479:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767726.769482:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775767726.769484:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767726.769485:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767726.769488:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767726.769490:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4979 00000400:00000200:3.0:1775767726.769494:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767726.769498:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767726.769502:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767726.769505:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767726.769509:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767726.769515:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767726.769517:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767726.769521:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767726.769523:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:3.0:1775767726.769526:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767726.769529:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:3.0:1775767726.769531:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:3.0:1775767726.769601:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:3.0:1775767726.769608:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767726.769611:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767726.769615:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767726.769617:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767726.769618:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00080000:3.0:1775767726.769623:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767726.769627:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767726.769628:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767726.769634:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767726.769639:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767726.769642:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767726.769648:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775767726.769651:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767726.769653:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767726.769658:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aa680. 00000020:00000040:3.0:1775767726.769661:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:3.0:1775767726.769665:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767726.769668:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775767726.769720:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767726.769723:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000100:00000001:3.0:1775767726.769764:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767726.769766:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767726.769768:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767726.769774:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0400. 00000100:00000001:3.0:1775767726.769778:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767726.769785:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aa680 x1862022294227328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767726.769795:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767726.769799:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767726.769804:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006cb1597d] to pc [ptlrpcd_rcv+-1] req@ffff9942099aa680 x1862022294227328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767726.769877:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767726.769881:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767726.769886:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775767726.769889:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767726.769892:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767726.769895:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767726.769900:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4979 00000400:00000200:3.0:1775767726.769909:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767726.769914:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767726.769919:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767726.769925:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767726.769928:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767726.769933:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767726.769937:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767726.769940:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767726.769942:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:3.0:1775767726.769945:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767726.769948:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:3.0:1775767726.769950:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:3.0:1775767726.769954:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:3.0:1775767726.769962:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767726.769967:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767726.769973:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767726.769978:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767726.769983:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:3.0:1775767726.769990:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767726.769996:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767726.770002:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767726.770008:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767726.770014:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767726.770019:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767726.770024:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775767726.770028:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767726.770031:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767726.770036:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ab800. 00000020:00000040:3.0:1775767726.770042:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:3.0:1775767726.770047:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767726.770048:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767726.770050:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767726.770053:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767726.770057:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767726.770061:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b3400. 00000100:00000001:3.0:1775767726.770066:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767726.770074:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ab800 x1862022294227456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767726.770089:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767726.770092:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767726.770098:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000089df80c0] to pc [ptlrpcd_rcv+-1] req@ffff9942099ab800 x1862022294227456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767726.770112:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767726.770118:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767726.770124:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775767726.770127:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767726.770129:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767726.770134:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767726.770136:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4979 00000400:00000200:3.0:1775767726.770143:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767726.770149:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767726.770152:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767726.770175:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767726.770178:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767726.770180:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767726.770183:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767726.770186:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767726.770189:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:3.0:1775767726.770192:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767726.770194:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:3.0:1775767726.770199:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:3.0:1775767726.770202:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:3.0:1775767726.770209:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767726.770212:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767726.770214:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767726.770218:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767726.770220:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:3.0:1775767726.770224:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767726.770227:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767726.770228:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767726.770232:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767726.770236:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767726.770237:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767726.770240:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:3.0:1775767726.770244:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767726.770246:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767726.770250:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aa300. 00000020:00000040:3.0:1775767726.770255:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:3.0:1775767726.770258:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767726.770259:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767726.770263:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767726.770266:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767726.770268:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767726.770271:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2000. 00000100:00000001:3.0:1775767726.770275:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767726.770283:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aa300 x1862022294227584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767726.770295:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767726.770299:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767726.770305:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f33bdf15] to pc [ptlrpcd_rcv+-1] req@ffff9942099aa300 x1862022294227584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767726.770316:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767726.770319:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767726.770324:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775767726.770328:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767726.770329:0:36:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767726.770334:0:36:0:(import.c:484:import_select_connection()) Process entered 00000800:00000200:2.1:1775767726.770337:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00080000:3.0:1775767726.770340:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4979 00000800:00000200:2.1:1775767726.770341:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000400:00000200:3.0:1775767726.770347:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767726.770355:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767726.770357:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767726.770365:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767726.770372:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767726.770379:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767726.770385:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767726.770392:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767726.770395:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:3.0:1775767726.770398:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767726.770403:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:3.0:1775767726.770408:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:3.0:1775767726.770414:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:3.0:1775767726.770422:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767726.770424:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767726.770428:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767726.770432:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767726.770437:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:3.0:1775767726.770442:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767726.770445:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767726.770449:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767726.770453:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767726.770457:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767726.770459:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767726.770462:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:3.0:1775767726.770465:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767726.770468:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767726.770472:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8a80. 00000020:00000040:3.0:1775767726.770476:0:36:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:3.0:1775767726.770481:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767726.770483:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767726.770486:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767726.770488:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767726.770491:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767726.770496:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b1000. 00000100:00000001:3.0:1775767726.770500:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767726.770507:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8a80 x1862022294227712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767726.770557:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767726.770561:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767726.770568:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006c3265af] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8a80 x1862022294227712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767726.770581:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767726.770587:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (4989) 00000800:00000200:2.1:1775767726.770736:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767726.770737:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775767726.770945:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767726.770947:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775767726.771122:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767726.771124:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000100:00000001:1.0:1775767727.793250:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767727.793258:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767727.793261:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767727.793268:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767727.793274:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767727.793279:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767727.793292:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767727.793292:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767727.793296:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767727.793297:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767727.793300:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767727.793300:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767727.793307:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767727.793311:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767727.793314:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767727.793317:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767727.793319:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767727.793322:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767728.113277:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299652925, at 4299652928 00000100:00000010:1.0:1775767728.113301:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d6ba0. 00000100:00000001:3.0:1775767728.817270:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767728.817271:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767728.817277:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767728.817277:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767728.817279:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767728.817281:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767728.817287:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775767728.817287:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767728.817293:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767728.817294:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767728.817299:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767728.817304:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767728.817305:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767728.817316:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767728.817320:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767728.817328:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767728.817331:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767728.817333:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767729.841236:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767729.841236:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767729.841242:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767729.841242:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767729.841245:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767729.841245:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767729.841252:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775767729.841252:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767729.841256:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767729.841256:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767729.841262:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767729.841263:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767729.841266:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767729.841270:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767729.841275:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767729.841275:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767729.841278:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767729.841280:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767730.865290:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767730.865292:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767730.865302:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767730.865303:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767730.865307:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767730.865313:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767730.865315:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767730.865321:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1775767730.865323:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767730.865326:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767730.865329:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767730.865330:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767730.865337:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767730.865339:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767730.865342:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767730.865345:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767730.865349:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767730.865354:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767731.889306:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767731.889315:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767731.889319:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767731.889322:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775767731.889332:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767731.889342:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:1.0:1775767731.889343:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775767731.889349:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767731.889353:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1775767731.889353:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767731.889359:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1775767731.889363:0:36:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767731.889367:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767731.889368:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775767731.889379:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 4984 00000400:00000200:1.0:1775767731.889397:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767731.889406:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775767731.889409:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775767731.889411:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767731.889416:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767731.889418:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767731.889420:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775767731.889425:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775767731.889427:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:1.0:1775767731.889431:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775767731.889434:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:1.0:1775767731.889438:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:1.0:1775767731.889441:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:1.0:1775767731.889447:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767731.889451:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767731.889456:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767731.889458:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775767731.889462:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:1.0:1775767731.889468:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775767731.889472:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767731.889475:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775767731.889479:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767731.889482:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767731.889485:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775767731.889503:0:36:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775767731.889513:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207888700. 00000020:00000040:1.0:1775767731.889517:0:36:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:1.0:1775767731.889520:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775767731.889522:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767731.889526:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775767731.889529:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775767731.889533:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775767731.889543:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543d800. 00000100:00000001:1.0:1775767731.889551:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767731.889561:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207888700 x1862022294227840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775767731.889572:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775767731.889577:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767731.889585:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009f2ba301] to pc [ptlrpcd_rcv+-1] req@ffff994207888700 x1862022294227840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767731.889611:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767731.889615:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767731.889619:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775767731.889621:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775767731.889623:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775767731.889626:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775767731.889630:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4984 00000400:00000200:1.0:1775767731.889636:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767731.889640:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775767731.889643:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775767731.889647:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767731.889662:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767731.889669:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767731.889675:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775767731.889681:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775767731.889684:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:1.0:1775767731.889688:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775767731.889692:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:1.0:1775767731.889697:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:1.0:1775767731.889703:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:1.0:1775767731.889711:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767731.889717:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767731.889720:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767731.889724:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775767731.889730:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00080000:1.0:1775767731.889735:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775767731.889742:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767731.889747:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775767731.889754:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767731.889759:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767731.889762:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775767731.889767:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775767731.889771:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767731.889774:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775767731.889779:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207889f80. 00000020:00000040:1.0:1775767731.889784:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:1.0:1775767731.889790:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775767731.889793:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767731.889798:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775767731.889801:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775767731.889805:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775767731.889812:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543f000. 00000100:00000001:1.0:1775767731.889817:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767731.889827:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207889f80 x1862022294227968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775767731.889843:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775767731.889848:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767731.889855:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ed2274f4] to pc [ptlrpcd_rcv+-1] req@ffff994207889f80 x1862022294227968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767731.889875:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767731.889881:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767731.889891:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775767731.889893:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775767731.889896:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775767731.889903:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775767731.889910:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4984 00000400:00000200:1.0:1775767731.889918:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767731.889925:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775767731.889928:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775767731.889935:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767731.889938:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767731.889940:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767731.889943:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775767731.889948:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775767731.889951:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:1.0:1775767731.889955:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775767731.889958:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000800:00000200:2.1:1775767731.889961:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000001:1.0:1775767731.889962:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000800:00000200:2.1:1775767731.889967:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000020:00000001:1.0:1775767731.889968:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:1.0:1775767731.889977:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767731.889982:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767731.889986:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767731.889990:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775767731.889996:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:1.0:1775767731.890002:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775767731.890009:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767731.890013:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775767731.890018:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767731.890025:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767731.890030:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775767731.890034:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775767731.890041:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767731.890047:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775767731.890055:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff99420788b100. 00000020:00000040:1.0:1775767731.890060:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:1.0:1775767731.890066:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775767731.890068:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767731.890070:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775767731.890074:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775767731.890077:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775767731.890084:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543dc00. 00000100:00000001:1.0:1775767731.890090:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767731.890097:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff99420788b100 x1862022294228096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775767731.890109:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775767731.890113:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767731.890124:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e26cf977] to pc [ptlrpcd_rcv+-1] req@ffff99420788b100 x1862022294228096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767731.890137:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767731.890142:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767731.890150:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775767731.890176:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775767731.890179:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775767731.890181:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775767731.890185:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4984 00000400:00000200:1.0:1775767731.890193:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767731.890199:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775767731.890201:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775767731.890206:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767731.890210:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767731.890214:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767731.890219:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775767731.890224:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775767731.890226:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:1.0:1775767731.890228:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775767731.890230:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:1.0:1775767731.890232:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:1.0:1775767731.890236:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:1.0:1775767731.890241:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767731.890245:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767731.890249:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767731.890253:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775767731.890257:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:1.0:1775767731.890260:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775767731.890263:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767731.890266:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775767731.890270:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767731.890273:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767731.890276:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775767731.890277:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:1.0:1775767731.890280:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767731.890284:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775767731.890289:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207888e00. 00000020:00000040:1.0:1775767731.890294:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:1.0:1775767731.890299:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775767731.890302:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767731.890305:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775767731.890308:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775767731.890312:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775767731.890320:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543d400. 00000100:00000001:1.0:1775767731.890327:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767731.890338:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207888e00 x1862022294228224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775767731.890358:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775767731.890363:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767731.890374:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001812f966] to pc [ptlrpcd_rcv+-1] req@ffff994207888e00 x1862022294228224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767731.890390:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767731.890395:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767731.890401:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775767731.890405:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775767731.890407:0:36:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775767731.890410:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775767731.890414:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4984 00000400:00000200:1.0:1775767731.890421:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767731.890424:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775767731.890428:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775767731.890433:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767731.890439:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767731.890442:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767731.890445:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775767731.890449:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775767731.890450:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:1.0:1775767731.890451:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775767731.890453:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:1.0:1775767731.890456:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:1.0:1775767731.890460:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:1.0:1775767731.890466:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767731.890468:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767731.890471:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767731.890475:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775767731.890477:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:1.0:1775767731.890481:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775767731.890482:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767731.890483:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775767731.890497:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767731.890500:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767731.890504:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775767731.890507:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:1.0:1775767731.890509:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767731.890512:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775767731.890518:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207889880. 00000020:00000040:1.0:1775767731.890521:0:36:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:1.0:1775767731.890524:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775767731.890526:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767731.890530:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775767731.890532:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775767731.890535:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775767731.890541:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543e400. 00000100:00000001:1.0:1775767731.890546:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767731.890554:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207889880 x1862022294228352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775767731.890568:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775767731.890573:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767731.890578:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f11856b8] to pc [ptlrpcd_rcv+-1] req@ffff994207889880 x1862022294228352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767731.890594:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767731.890599:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (4994) 00000800:00000200:2.1:1775767731.890673:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767731.890676:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775767731.890909:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767731.890912:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775767731.891145:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767731.891147:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775767731.891523:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767731.891526:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000100:00000001:3.0:1775767732.913255:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767732.913264:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767732.913269:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767732.913279:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767732.913288:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767732.913294:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767732.913317:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767732.913320:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767732.913324:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767733.233270:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299658045, at 4299658048 00000100:00000010:1.0:1775767733.233284:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d6f60. 00000100:00000001:3.0:1775767733.937225:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767733.937233:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767733.937237:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767733.937245:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767733.937250:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767733.937254:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767733.937265:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767733.937268:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767733.937271:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767734.961267:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767734.961275:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767734.961279:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767734.961283:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:3.0:1775767734.961288:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767734.961290:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767734.961294:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767734.961294:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1775767734.961300:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767734.961301:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767734.961307:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767734.961311:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767734.961316:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767734.961321:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767734.961326:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767734.961327:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767734.961330:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767734.961333:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767735.985211:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767735.985217:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767735.985221:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767735.985222:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767735.985223:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767735.985225:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767735.985232:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775767735.985233:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767735.985239:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767735.985240:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767735.985244:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767735.985245:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767735.985248:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767735.985251:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767735.985253:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767735.985254:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767735.985257:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767735.985261:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767737.009265:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767737.009272:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775767737.009273:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767737.009278:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767737.009278:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775767737.009282:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767737.009284:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775767737.009286:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767737.009292:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767737.009293:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775767737.009295:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775767737.009303:0:13471:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 4989 00000100:00000001:1.0:1775767737.009308:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767737.009311:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767737.009317:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767737.009317:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767737.009327:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767737.009331:0:13471:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767737.009337:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767737.009342:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767737.009347:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767737.009351:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767737.009357:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767737.009360:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:3.0:1775767737.009362:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767737.009365:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:3.0:1775767737.009369:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:3.0:1775767737.009373:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:3.0:1775767737.009381:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767737.009384:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767737.009397:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767737.009402:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767737.009406:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:3.0:1775767737.009410:0:13471:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767737.009413:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767737.009417:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767737.009420:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767737.009424:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767737.009426:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775767737.009429:0:13471:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767737.009453:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8a80. 00000020:00000040:3.0:1775767737.009458:0:13471:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:3.0:1775767737.009462:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767737.009464:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767737.009468:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767737.009471:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767737.009476:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767737.009486:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2c00. 00000100:00000001:3.0:1775767737.009493:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767737.009502:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8a80 x1862022294228480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767737.009513:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767737.009518:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767737.009523:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006c3265af] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8a80 x1862022294228480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767737.009543:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767737.009547:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767737.009550:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775767737.009552:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767737.009554:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767737.009557:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767737.009560:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4989 00000400:00000200:3.0:1775767737.009566:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767737.009570:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767737.009573:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767737.009577:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767737.009581:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767737.009586:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767737.009590:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767737.009593:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767737.009595:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:3.0:1775767737.009597:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767737.009601:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:3.0:1775767737.009606:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:3.0:1775767737.009609:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:3.0:1775767737.009613:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767737.009619:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767737.009623:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767737.009627:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767737.009633:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00080000:3.0:1775767737.009638:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767737.009643:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767737.009648:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767737.009657:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767737.009661:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767737.009664:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767737.009677:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775767737.009687:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767737.009690:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767737.009698:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aa300. 00000020:00000040:3.0:1775767737.009703:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:3.0:1775767737.009707:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767737.009711:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767737.009713:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767737.009715:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767737.009719:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767737.009725:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0c00. 00000100:00000001:3.0:1775767737.009730:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767737.009741:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aa300 x1862022294228608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767737.009771:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767737.009777:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767737.009786:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f33bdf15] to pc [ptlrpcd_rcv+-1] req@ffff9942099aa300 x1862022294228608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767737.009804:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767737.009808:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767737.009814:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775767737.009817:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767737.009819:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767737.009824:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767737.009831:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4989 00000400:00000200:3.0:1775767737.009857:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767737.009866:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767737.009872:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767737.009879:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767737.009883:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767737.009887:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767737.009890:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767737.009894:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767737.009897:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:3.0:1775767737.009900:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767737.009903:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:3.0:1775767737.009908:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000800:00000200:2.1:1775767737.009910:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000001:3.0:1775767737.009912:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:3.0:1775767737.009917:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767737.009919:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767737.009924:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767737.009927:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000800:00000200:2.1:1775767737.009927:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000020:00000040:3.0:1775767737.009932:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:3.0:1775767737.009940:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767737.009947:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767737.009950:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767737.009953:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767737.009960:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767737.009965:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767737.009967:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775767737.009972:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767737.009975:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767737.009981:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ab800. 00000020:00000040:3.0:1775767737.009986:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:3.0:1775767737.009991:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767737.009994:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767737.009999:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767737.010001:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767737.010004:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767737.010010:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2800. 00000100:00000001:3.0:1775767737.010016:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767737.010024:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ab800 x1862022294228736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767737.010046:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767737.010050:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767737.010058:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000089df80c0] to pc [ptlrpcd_rcv+-1] req@ffff9942099ab800 x1862022294228736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767737.010070:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767737.010076:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767737.010083:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775767737.010087:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767737.010089:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767737.010094:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767737.010098:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4989 00000400:00000200:3.0:1775767737.010104:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767737.010109:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767737.010112:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767737.010117:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767737.010120:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767737.010124:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767737.010128:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767737.010132:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767737.010135:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:3.0:1775767737.010137:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767737.010141:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:3.0:1775767737.010144:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:3.0:1775767737.010149:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:3.0:1775767737.010177:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767737.010181:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767737.010184:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767737.010186:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767737.010189:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:3.0:1775767737.010194:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767737.010199:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767737.010202:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767737.010207:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767737.010213:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767737.010219:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767737.010223:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:3.0:1775767737.010227:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767737.010229:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767737.010235:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aa680. 00000020:00000040:3.0:1775767737.010240:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:3.0:1775767737.010244:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767737.010247:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767737.010251:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767737.010253:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767737.010258:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767737.010264:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0000. 00000100:00000001:3.0:1775767737.010270:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767737.010279:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aa680 x1862022294228864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767737.010298:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767737.010302:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767737.010312:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006cb1597d] to pc [ptlrpcd_rcv+-1] req@ffff9942099aa680 x1862022294228864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767737.010331:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767737.010335:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767737.010342:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775767737.010345:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767737.010348:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767737.010353:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767737.010358:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 4989 00000400:00000200:3.0:1775767737.010365:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767737.010371:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767737.010374:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767737.010383:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767737.010388:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767737.010391:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767737.010396:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767737.010399:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767737.010401:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:3.0:1775767737.010405:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767737.010409:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:3.0:1775767737.010414:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:3.0:1775767737.010418:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:3.0:1775767737.010424:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767737.010426:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767737.010428:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767737.010430:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767737.010433:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:3.0:1775767737.010451:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767737.010456:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767737.010458:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767737.010463:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767737.010468:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767737.010474:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767737.010476:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:3.0:1775767737.010481:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767737.010484:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767737.010491:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aad80. 00000020:00000040:3.0:1775767737.010496:0:13471:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:3.0:1775767737.010501:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767737.010504:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767737.010508:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767737.010511:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767737.010514:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767737.010519:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b1000. 00000100:00000001:3.0:1775767737.010526:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767737.010535:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aad80 x1862022294228992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767737.010562:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767737.010565:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767737.010571:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000045ccf08] to pc [ptlrpcd_rcv+-1] req@ffff9942099aad80 x1862022294228992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767737.010585:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767737.010591:0:13471:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (4999) 00000800:00000200:2.1:1775767737.010749:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767737.010752:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775767737.011029:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767737.011032:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775767737.011357:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767737.011360:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775767737.011682:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767737.011684:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000100:00000001:1.0:1775767738.033231:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767738.033239:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767738.033242:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767738.033250:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767738.033256:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767738.033261:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767738.033277:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767738.033279:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767738.033283:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767738.353267:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299663165, at 4299663168 00000100:00000010:1.0:1775767738.353283:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d67e0. 00000100:00000001:1.0:1775767739.057272:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767739.057280:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767739.057284:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767739.057289:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767739.057293:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767739.057296:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767739.057298:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767739.057299:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767739.057301:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767740.081253:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767740.081261:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767740.081265:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767740.081271:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767740.081278:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767740.081282:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767740.081292:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767740.081294:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767740.081297:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767741.105237:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767741.105242:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767741.105244:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767741.105249:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767741.105254:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767741.105257:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767741.105258:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767741.105263:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767741.105266:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767741.105269:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00100000:3.0:1775767741.105269:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767741.105273:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767741.105275:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767741.105276:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767741.105278:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767741.105280:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767741.105281:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767741.105284:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767742.129253:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767742.129253:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767742.129259:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767742.129260:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767742.129262:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767742.129264:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767742.129268:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775767742.129269:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767742.129272:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767742.129276:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767742.129277:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767742.129280:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767742.129281:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767742.129285:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767742.129288:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767742.129288:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767742.129290:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767742.129293:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775767742.129785:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767742.129789:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775767742.130204:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767742.130205:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775767742.130413:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767742.130415:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775767742.130591:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767742.130593:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775767742.130802:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767742.130804:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000100:2.0:1775767742.130806:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767742.130808:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3a00. 00000400:00000200:2.0:1775767742.130810:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775767742.130812:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775767742.130814:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f990 00000400:00000010:2.0:1775767742.130815:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f990. 00000100:00000001:2.0:1775767742.130816:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767742.130818:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fd880 x1862022294229632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767767 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767742.130824:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767742.130825:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fd880 x1862022294229632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767767 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767742.130830:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767742.130830:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775767742.130832:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767742.130834:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:2.0:1775767742.130836:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767742.130837:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299666981 00000100:00000001:2.0:1775767742.130838:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299668285 00000100:00000001:2.0:1775767742.130840:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299666981 00000100:00000001:2.0:1775767742.130841:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299668285 00000100:00000001:2.0:1775767742.130843:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767742.130844:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767742.130845:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767742.130848:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767742/real 1775767742] req@ffff9942099fd880 x1862022294229632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775767742.130852:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767742.130853:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767742.130854:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b391b8. 00000400:00000200:2.0:1775767742.130856:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f770 00000400:00000010:2.0:1775767742.130857:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f770. 00000100:00000001:2.0:1775767742.130858:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767742.130860:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fd880 x1862022294229632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767742.130866:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fd880 x1862022294229632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767742.130869:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767742.130870:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767742.130871:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767742.130872:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767742.130874:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fd880 x1862022294229632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767742.130878:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767742.130880:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fd880 x1862022294229632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767767 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767742.130884:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767742.130886:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767742.130887:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767742.130887:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767742.130888:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767742.130891:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767742.130892:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767742.130894:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767742.130896:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767742.130898:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fd880 x1862022294229632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767767 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767742.130903:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294229632:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767742.130905:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767742.130906:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767742.130909:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fed80 x1862022294229248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767767 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767742.130913:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767742.130914:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767742.130915:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1c800. 02000000:00000001:2.0:1775767742.130917:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767742.130918:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767742.130918:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:2.0:1775767742.130920:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767742.130921:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002a800. 02000000:00000001:2.0:1775767742.130923:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767742.130924:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767742.130925:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fed80. 00000100:00000001:2.0:1775767742.130927:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767742.130928:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767742.130929:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767742.130931:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fd180 x1862022294229376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767767 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767742.130935:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767742.130936:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767742.130937:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1e800. 02000000:00000001:2.0:1775767742.130939:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767742.130939:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767742.130940:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:2.0:1775767742.130941:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767742.130942:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002b800. 02000000:00000001:2.0:1775767742.130943:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767742.130944:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767742.130945:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fd180. 00000100:00000001:2.0:1775767742.130946:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767742.130947:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767742.130948:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767742.130950:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ff480 x1862022294229504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767767 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767742.130954:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767742.130955:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767742.130956:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f000. 02000000:00000001:2.0:1775767742.130957:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767742.130958:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767742.130959:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:2.0:1775767742.130960:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767742.130961:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002bc00. 02000000:00000001:2.0:1775767742.130962:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767742.130963:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767742.130963:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ff480. 00000100:00000001:2.0:1775767742.130965:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767742.130965:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767742.130966:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767742.130968:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fd880 x1862022294229632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767767 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767742.130972:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767742.130972:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767742.130973:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f400. 02000000:00000001:2.0:1775767742.130974:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767742.130975:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767742.130975:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:2.0:1775767742.130976:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767742.130977:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029400. 02000000:00000001:2.0:1775767742.130978:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767742.130979:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767742.130980:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fd880. 00000100:00000001:2.0:1775767742.130981:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767742.130981:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767742.130982:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767742.130985:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767742.130986:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767742.130987:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775767742.130988:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767742.193313:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767742.193316:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767742.193319:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775767743.153232:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767743.153233:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767743.153238:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767743.153239:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767743.153242:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767743.153242:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767743.153249:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775767743.153249:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767743.153254:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767743.153255:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767743.153260:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767743.153260:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767743.153266:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767743.153268:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767743.153273:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767743.153273:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767743.153275:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767743.153281:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767743.153282:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767743.153287:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767743.153289:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767743.153295:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775767743.153300:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767743.153303:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767743.153305:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767743.153307:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767743.153310:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767743.473246:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299668285, at 4299668288 00000100:00000010:1.0:1775767743.473264:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d6cc0. 00000100:00000001:1.0:1775767744.177258:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767744.177258:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767744.177264:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767744.177265:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767744.177267:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767744.177268:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767744.177272:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775767744.177274:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767744.177277:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767744.177278:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767744.177279:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767744.177281:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767744.177287:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767744.177287:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767744.177288:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767744.177293:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767744.177294:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767744.177305:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767744.177306:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767744.177307:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767744.177312:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767744.177313:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767744.177318:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767744.177319:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767744.177325:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767744.177327:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767744.177329:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767744.241396:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767744.241401:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767744.241405:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775767745.201234:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767745.201234:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767745.201234:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767745.201241:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767745.201241:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767745.201242:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767745.201244:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767745.201245:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767745.201246:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767745.201253:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775767745.201253:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775767745.201254:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767745.201258:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767745.201259:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767745.201260:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767745.201262:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767745.201263:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767745.201264:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767745.201268:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767745.201271:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767745.201273:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767745.201275:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767745.201278:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767745.201280:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767745.201282:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767745.201285:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767745.201289:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767746.225249:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767746.225249:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767746.225250:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767746.225260:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767746.225260:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767746.225260:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767746.225265:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767746.225265:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767746.225266:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767746.225275:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:1.0:1775767746.225276:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775767746.225276:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767746.225285:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767746.225285:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767746.225285:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767746.225292:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767746.225292:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767746.225293:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767746.225300:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767746.225303:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767746.225307:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767746.225308:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767746.225312:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767746.225315:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767746.225317:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767746.225321:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767746.225324:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767746.289416:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767746.289420:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767746.289423:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775767747.249270:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767747.249277:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767747.249279:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767747.249284:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767747.249284:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767747.249289:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767747.249298:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775767747.249299:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000080:00000001:1.0:1775767747.249303:0:36:0:(rw.c:1481:ll_writepages()) Process entered 00000100:00000001:2.0:1775767747.249307:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000080:00000001:1.0:1775767747.249309:0:36:0:(file.c:5250:cl_sync_file_range()) Process entered 00000100:00000001:3.0:1775767747.249309:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767747.249312:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1775767747.249314:0:36:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00000001:2.0:1775767747.249317:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767747.249318:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767747.249320:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767747.249324:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767747.249325:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000010:1.0:1775767747.249327:0:36:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994206aba240. 00000100:00000001:3.0:1775767747.249329:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767747.249335:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1775767747.249337:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994208a8f600. 00000020:00000010:1.0:1775767747.249361:0:36:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994208a8e000. 00000100:00000001:2.0:1775767747.249362:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000020:00000010:1.0:1775767747.249366:0:36:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207cba360. 00000100:00000001:2.0:1775767747.249366:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767747.249370:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767747.249374:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000008:00000010:1.0:1775767747.249377:0:36:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207e02ca0. 00000100:00000001:2.0:1775767747.249378:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767747.249382:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00020000:00000010:1.0:1775767747.249385:0:36:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff9942031b3480. 00000100:00000001:2.0:1775767747.249385:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767747.249388:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00008000:00000010:1.0:1775767747.249392:0:36:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942054ab170. 00000100:00000001:2.0:1775767747.249393:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:1.0:1775767747.249401:0:36:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99422030bf18. 00000080:00000010:1.0:1775767747.249408:0:36:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff994205751da8. 00000020:00000010:1.0:1775767747.249413:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994208a8ec00. 00000008:00000010:1.0:1775767747.249418:0:36:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff994210709ce0. 00020000:00000010:1.0:1775767747.249422:0:36:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994205743180. 00008000:00000010:1.0:1775767747.249425:0:36:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994219fec110. 00000080:00000010:1.0:1775767747.249431:0:36:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942057e2960. 00000020:00000001:1.0:1775767747.249436:0:36:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107591643720 : -112966117907896 : ffff994206aba248) 00000020:00001000:1.0:1775767747.249438:0:36:0:(cl_object.c:831:cl_env_get()) 1@ffff994206aba240 00000020:00000001:1.0:1775767747.249443:0:36:0:(cl_object.c:151:cl_object_top()) ffff99420791a5f8 -> ffff99420791a5f8 00000020:00000001:1.0:1775767747.249446:0:36:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:1.0:1775767747.249449:0:36:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:1.0:1775767747.249452:0:36:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x10176:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:1.0:1775767747.249457:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:1.0:1775767747.249460:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:1.0:1775767747.249462:0:36:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:1.0:1775767747.249469:0:36:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x10176:0x0]io ffff994205751e08 type 6 ignore/verify layout 1/0 00020000:00000002:1.0:1775767747.249476:0:36:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207dfb6c8) owner 0000000000000000/ffff994201348000 00020000:00000001:1.0:1775767747.249480:0:36:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:1.0:1775767747.249482:0:36:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:1.0:1775767747.249486:0:36:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00020000:00000001:1.0:1775767747.249488:0:36:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775767747.249492:0:36:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:1.0:1775767747.249495:0:36:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:1.0:1775767747.249498:0:36:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775767747.249501:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:1.0:1775767747.249503:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:1.0:1775767747.249504:0:36:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00020000:00000002:1.0:1775767747.249507:0:36:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207dfb6c8) owner 0000000000000000/ffff994201348000 00000020:00000001:1.0:1775767747.249511:0:36:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775767747.249514:0:36:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:1.0:1775767747.249515:0:36:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00020000:00000001:1.0:1775767747.249517:0:36:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00020000:00200000:1.0:1775767747.249520:0:36:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00000002:1.0:1775767747.249524:0:36:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:1.0:1775767747.249530:0:36:0:(lov_io.c:155:lov_sub_get()) Process entered 00020000:00000001:1.0:1775767747.249532:0:36:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000020:00000001:1.0:1775767747.249533:0:36:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:1.0:1775767747.249535:0:36:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994206ababd0. 00000020:00000010:1.0:1775767747.249539:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994201343000. 00000020:00000010:1.0:1775767747.249544:0:36:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994201342e00. 00000020:00000010:1.0:1775767747.249547:0:36:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207cbaae0. 00000008:00000010:1.0:1775767747.249551:0:36:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207e00000. 00020000:00000010:1.0:1775767747.249554:0:36:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff9942031b30c0. 00008000:00000010:1.0:1775767747.249558:0:36:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942054a8a98. 00000080:00000010:1.0:1775767747.249562:0:36:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99422030b0a8. 00000080:00000010:1.0:1775767747.249565:0:36:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff994205753d98. 00000020:00000010:1.0:1775767747.249569:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994201343200. 00000008:00000010:1.0:1775767747.249574:0:36:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff994210709080. 00020000:00000010:1.0:1775767747.249578:0:36:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff9942057418c0. 00008000:00000010:1.0:1775767747.249583:0:36:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994219fec368. 00000080:00000010:1.0:1775767747.249587:0:36:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942057e24b0. 00000020:00000001:1.0:1775767747.249589:0:36:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107591646168 : -112966117905448 : ffff994206ababd8) 00000020:00001000:1.0:1775767747.249590:0:36:0:(cl_object.c:831:cl_env_get()) 1@ffff994206ababd0 00000020:00000001:1.0:1775767747.249592:0:36:0:(cl_object.c:151:cl_object_top()) ffff99420791ae90 -> ffff99420791a5f8 00000020:00000001:1.0:1775767747.249596:0:36:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:1.0:1775767747.249599:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:1.0:1775767747.249601:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:1.0:1775767747.249603:0:36:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775767747.249605:0:36:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775767747.249608:0:36:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107571233256 : -112966138318360 : ffff9942057431e8) 00000020:00000001:1.0:1775767747.249611:0:36:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000020:00000001:1.0:1775767747.249614:0:36:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775767747.249616:0:36:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775767747.249619:0:36:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775767747.249620:0:36:0:(cl_io.c:314:cl_io_lock()) Process entered 00020000:00000001:1.0:1775767747.249621:0:36:0:(lov_io.c:1108:lov_io_lock()) Process entered 00020000:00000001:1.0:1775767747.249623:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:1.0:1775767747.249625:0:36:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:1.0:1775767747.249627:0:36:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:1.0:1775767747.249628:0:36:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775767747.249629:0:36:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775767747.249630:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775767747.249632:0:36:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775767747.249633:0:36:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:1.0:1775767747.249635:0:36:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775767747.249637:0:36:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775767747.249639:0:36:0:(cl_io.c:528:cl_io_start()) Process entered 00020000:00000001:1.0:1775767747.249640:0:36:0:(lov_io.c:1114:lov_io_start()) Process entered 00020000:00000001:1.0:1775767747.249641:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:1.0:1775767747.249642:0:36:0:(cl_io.c:528:cl_io_start()) Process entered 00000008:00000001:1.0:1775767747.249645:0:36:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000008:00000001:1.0:1775767747.249647:0:36:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000008:00000001:1.0:1775767747.249654:0:36:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:1.0:1775767747.249656:0:36:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:1.0:1775767747.249657:0:36:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:1.0:1775767747.249659:0:36:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000008:00000001:1.0:1775767747.249661:0:36:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000008:00000001:1.0:1775767747.249664:0:36:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107754940544 : -112965954611072 : ffff994210675880) 00000008:00000020:1.0:1775767747.249670:0:36:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994210675880 ready 0|-|- wr 1|-|+ rd 0|- 10 in flight 00000008:00000001:1.0:1775767747.249675:0:36:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:1.0:1775767747.249677:0:36:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:1.0:1775767747.249680:0:36:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:1.0:1775767747.249681:0:36:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000008:00000020:1.0:1775767747.249685:0:36:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994218232fc0 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000008:00000001:1.0:1775767747.249690:0:36:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:1.0:1775767747.249692:0:36:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775767747.249696:0:36:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775767747.249701:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775767747.249704:0:36:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775767747.249708:0:36:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775767747.249709:0:36:0:(cl_io.c:555:cl_io_end()) Process entered 00020000:00000001:1.0:1775767747.249711:0:36:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00020000:00000001:1.0:1775767747.249712:0:36:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000020:00000001:1.0:1775767747.249713:0:36:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000001:1.0:1775767747.249714:0:36:0:(cl_io.c:563:cl_io_end()) Process leaving 00020000:00000001:1.0:1775767747.249715:0:36:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775767747.249716:0:36:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000020:00000001:1.0:1775767747.249717:0:36:0:(cl_io.c:563:cl_io_end()) Process leaving 00000020:00000001:1.0:1775767747.249720:0:36:0:(cl_io.c:352:cl_io_unlock()) Process entered 00020000:00000001:1.0:1775767747.249723:0:36:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00020000:00000001:1.0:1775767747.249725:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:1.0:1775767747.249727:0:36:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000020:00000001:1.0:1775767747.249728:0:36:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:1.0:1775767747.249731:0:36:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775767747.249734:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775767747.249736:0:36:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000020:00000001:1.0:1775767747.249738:0:36:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:1.0:1775767747.249740:0:36:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000020:00000001:1.0:1775767747.249743:0:36:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000001:1.0:1775767747.249745:0:36:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:1.0:1775767747.249747:0:36:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00020000:00000001:1.0:1775767747.249749:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:1.0:1775767747.249751:0:36:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000001:1.0:1775767747.249754:0:36:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00020000:00000001:1.0:1775767747.249755:0:36:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775767747.249758:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775767747.249761:0:36:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000020:00000001:1.0:1775767747.249763:0:36:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000020:00000001:1.0:1775767747.249766:0:36:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775767747.249770:0:36:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:1.0:1775767747.249773:0:36:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:1.0:1775767747.249775:0:36:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000100:00000001:2.0:1775767747.249776:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1775767747.249778:0:36:0:(cl_io.c:71:cl_io_fini()) Process entered 00000100:00000001:2.0:1775767747.249780:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:1.0:1775767747.249781:0:36:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000100:00000001:2.0:1775767747.249784:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00001000:1.0:1775767747.249785:0:36:0:(cl_object.c:925:cl_env_put()) 1@ffff994206ababd0 00000020:00000010:1.0:1775767747.249791:0:36:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994201342e00. 00000100:00000040:2.0:1775767747.249794:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fe300 x1862022294229760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1775767747.249797:0:36:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207cbaae0. 00000008:00000010:1.0:1775767747.249803:0:36:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207e00000. 00020000:00000010:1.0:1775767747.249810:0:36:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff9942031b30c0. 00000100:00000001:2.0:1775767747.249813:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767747.249815:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:1.0:1775767747.249816:0:36:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942054a8a98. 02000000:00000001:2.0:1775767747.249820:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000080:00000010:1.0:1775767747.249821:0:36:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99422030b0a8. 00000080:00000010:1.0:1775767747.249826:0:36:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff994205753d98. 02000000:00000001:2.0:1775767747.249826:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1775767747.249833:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994201343000. 02000000:00000001:2.0:1775767747.249833:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:1.0:1775767747.249839:0:36:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff994210709080. 00000100:00100000:2.0:1775767747.249842:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294229760:192.168.202.156@tcp:250:kworker.0 00020000:00000010:1.0:1775767747.249844:0:36:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff9942057418c0. 00008000:00000010:1.0:1775767747.249851:0:36:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994219fec368. 00000100:00000001:2.0:1775767747.249851:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000080:00000010:1.0:1775767747.249856:0:36:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942057e24b0. 02000000:00000001:2.0:1775767747.249856:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000020:00000010:1.0:1775767747.249861:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994201343200. 02000000:00000001:2.0:1775767747.249861:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1775767747.249867:0:36:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994206ababd0. 02000000:00000001:2.0:1775767747.249868:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00020000:00000001:1.0:1775767747.249871:0:36:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:1.0:1775767747.249874:0:36:0:(lov_io.c:694:lov_io_fini()) Process leaving 02000000:00000010:2.0:1775767747.249875:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f400. 00000080:00000001:1.0:1775767747.249876:0:36:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000080:00200000:1.0:1775767747.249880:0:36:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x10176:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 02000000:00000001:2.0:1775767747.249880:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1775767747.249887:0:36:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:1.0:1775767747.249889:0:36:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000400:00000200:2.0:1775767747.249891:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00001000:1.0:1775767747.249893:0:36:0:(cl_object.c:925:cl_env_put()) 1@ffff994206aba240 00000020:00000010:1.0:1775767747.249897:0:36:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994208a8e000. 00000400:00000010:2.0:1775767747.249897:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b391b8. 00000020:00000010:1.0:1775767747.249903:0:36:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207cba360. 00000008:00000010:1.0:1775767747.249907:0:36:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207e02ca0. 00000400:00000010:2.0:1775767747.249907:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f770. 00020000:00000010:1.0:1775767747.249914:0:36:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff9942031b3480. 00000100:00000200:2.0:1775767747.249917:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294229760, portal 25 00008000:00000010:1.0:1775767747.249918:0:36:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942054ab170. 00000080:00000010:1.0:1775767747.249924:0:36:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99422030bf18. 00000100:00000001:2.0:1775767747.249924:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767747.249927:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641205504 : -112966068346112 : ffff9942099fe300) 00000080:00000010:1.0:1775767747.249929:0:36:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff994205751da8. 00000020:00000010:1.0:1775767747.249933:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994208a8f600. 00000100:00000040:2.0:1775767747.249937:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fe300 x1862022294229760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767772 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000010:1.0:1775767747.249939:0:36:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff994210709ce0. 00020000:00000010:1.0:1775767747.249945:0:36:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994205743180. 00008000:00000010:1.0:1775767747.249951:0:36:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994219fec110. 00000100:00000001:2.0:1775767747.249955:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000080:00000010:1.0:1775767747.249956:0:36:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942057e2960. 00000020:00000010:1.0:1775767747.249962:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994208a8ec00. 00000100:00000040:2.0:1775767747.249963:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000020:00000010:1.0:1775767747.249966:0:36:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994206aba240. 00000400:00000010:2.0:1775767747.249968:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f990. 00000080:00000001:1.0:1775767747.249972:0:36:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:2.0:1775767747.249974:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294229760, offset 0 00000080:00000001:1.0:1775767747.249977:0:36:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767747.249984:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767747.249996:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767747.250004:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767747.250014:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767747.250019:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767747.250025:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767747.250030:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767747.250037:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224968:224968:256:4294967295] 192.168.202.156@tcp LPNI seq info [224964:224964:8:4294967295] 00000400:00000200:2.0:1775767747.250046:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767747.250060:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767747.250070:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767747.250079:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3a00. 00000800:00000200:2.0:1775767747.250094:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775767747.250100:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775767747.250114:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767747.250121:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000100:2.0:1775767747.250133:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767747.250137:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3a00. 00000400:00000200:2.0:1775767747.250145:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775767747.250173:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775767747.250179:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f990 00000400:00000010:2.0:1775767747.250181:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f990. 00000100:00000001:2.0:1775767747.250185:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767747.250191:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fe300 x1862022294229760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767772 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767747.250208:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767747.250211:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fe300 x1862022294229760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767772 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767747.250219:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767747.250221:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775767747.250229:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767747.250236:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767747.250241:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299673406 00000100:00000010:2.0:1775767747.250245:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d6900. 00000100:00000001:2.0:1775767747.250255:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1775767747.250262:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299673406 00000100:00000010:2.0:1775767747.250267:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d6660. 00000100:00000001:2.0:1775767747.250270:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1775767747.250275:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767747.250280:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767747.250284:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767747.250290:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767747/real 1775767747] req@ffff9942099fe300 x1862022294229760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775767747.250306:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767747.250309:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767747.250312:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b391b8. 00000400:00000200:2.0:1775767747.250315:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f770 00000400:00000010:2.0:1775767747.250319:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f770. 00000100:00000001:2.0:1775767747.250324:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767747.250329:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fe300 x1862022294229760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767747.250379:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fe300 x1862022294229760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767747.250394:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767747.250397:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767747.250401:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767747.250404:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767747.250412:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fe300 x1862022294229760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767747.250426:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767747.250433:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fe300 x1862022294229760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767772 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767747.250453:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767747.250455:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767747.250458:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767747.250460:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767747.250465:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767747.250468:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767747.250473:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767747.250482:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767747.250487:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767747.250494:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fe300 x1862022294229760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767772 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767747.250515:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294229760:192.168.202.156@tcp:250:kworker.0 00000100:00000001:2.0:1775767747.250524:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767747.250528:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767747.250532:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fe300 x1862022294229760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767772 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767747.250561:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767747.250565:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767747.250569:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f400. 02000000:00000001:2.0:1775767747.250573:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767747.250576:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767747.250580:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:2.0:1775767747.250584:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767747.250587:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028400. 02000000:00000001:2.0:1775767747.250593:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767747.250596:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767747.250599:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fe300. 00000100:00000001:2.0:1775767747.250604:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767747.250607:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767747.250609:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767747.250615:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767747.250618:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767747.250621:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775767747.250624:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767747.250626:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767747.250631:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fc700 x1862022294229888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767747.250643:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767747.250645:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767747.250648:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767747.250652:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767747.250656:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767747.250660:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294229888:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775767747.250664:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767747.250667:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767747.250669:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767747.250672:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775767747.250677:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f400. 02000000:00000001:2.0:1775767747.250680:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767747.250687:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767747.250691:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b391b8. 00000400:00000010:2.0:1775767747.250697:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f770. 00000100:00000200:2.0:1775767747.250701:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294229888, portal 10 00000100:00000001:2.0:1775767747.250703:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767747.250705:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641198336 : -112966068353280 : ffff9942099fc700) 00000100:00000040:2.0:1775767747.250714:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fc700 x1862022294229888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767772 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767747.250719:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767747.250722:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767747.250723:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f990. 00000100:00000200:2.0:1775767747.250725:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294229888, offset 0 00000400:00000200:2.0:1775767747.250728:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767747.250730:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767747.250733:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767747.250736:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767747.250738:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767747.250739:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767747.250741:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767747.250743:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224969:224969:256:4294967295] 192.168.202.156@tcp LPNI seq info [224965:224965:8:4294967295] 00000400:00000200:2.0:1775767747.250746:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767747.250751:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767747.250754:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767747.250756:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3a00. 00000800:00000200:2.0:1775767747.250759:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775767747.250761:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775767747.250764:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767747.250766:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000100:2.0:1775767747.250769:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767747.250770:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3a00. 00000400:00000200:2.0:1775767747.250772:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775767747.250775:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775767747.250777:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f990 00000400:00000010:2.0:1775767747.250778:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f990. 00000100:00000001:2.0:1775767747.250781:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767747.250783:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fc700 x1862022294229888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767772 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767747.250789:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767747.250791:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fc700 x1862022294229888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767772 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767747.250795:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767747.250797:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775767747.250799:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767747.250801:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:2.0:1775767747.250804:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767747.250805:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299672083 00000100:00000001:2.0:1775767747.250807:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299673406 00000100:00000001:2.0:1775767747.250809:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299672083 00000100:00000001:2.0:1775767747.250810:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299673406 00000100:00000001:2.0:1775767747.250812:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767747.250813:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767747.250815:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767747.250817:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767747/real 1775767747] req@ffff9942099fc700 x1862022294229888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775767747.250821:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767747.250822:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767747.250823:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b391b8. 00000400:00000200:2.0:1775767747.250824:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f770 00000400:00000010:2.0:1775767747.250825:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f770. 00000100:00000001:2.0:1775767747.250826:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767747.250828:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fc700 x1862022294229888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767747.250833:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fc700 x1862022294229888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767747.250842:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767747.250843:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767747.250844:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767747.250845:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767747.250848:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fc700 x1862022294229888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767747.250851:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767747.250854:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fc700 x1862022294229888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767772 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767747.250858:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767747.250859:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767747.250860:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767747.250861:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767747.250862:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767747.250864:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767747.250866:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767747.250868:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767747.250869:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767747.250871:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fc700 x1862022294229888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767772 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767747.250876:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294229888:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775767747.250878:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767747.250880:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ff100 x1862022294230016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767747.250885:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767747.250886:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767747.250887:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767747.250889:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767747.250890:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767747.250892:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294230016:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775767747.250894:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767747.250895:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767747.250896:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767747.250897:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775767747.250899:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f000. 02000000:00000001:2.0:1775767747.250900:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767747.250902:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767747.250912:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b391b8. 00000400:00000010:2.0:1775767747.250915:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f770. 00000100:00000200:2.0:1775767747.250916:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294230016, portal 10 00000100:00000001:2.0:1775767747.250921:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767747.250922:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641209088 : -112966068342528 : ffff9942099ff100) 00000100:00000040:2.0:1775767747.250926:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ff100 x1862022294230016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767772 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767747.250930:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767747.250932:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767747.250933:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f990. 00000100:00000200:2.0:1775767747.250935:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294230016, offset 0 00000400:00000200:2.0:1775767747.250937:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767747.250939:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767747.250941:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767747.250944:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767747.250946:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767747.250948:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767747.250950:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767747.250953:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224970:224970:256:4294967295] 192.168.202.156@tcp LPNI seq info [224966:224966:8:4294967295] 00000400:00000200:2.0:1775767747.250958:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767747.250964:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767747.250971:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767747.250974:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3a00. 00000800:00000200:2.0:1775767747.250978:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775767747.250980:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775767747.250984:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767747.250986:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000100:2.0:1775767747.250989:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767747.250991:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3a00. 00000400:00000200:2.0:1775767747.250994:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775767747.250997:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775767747.251000:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f990 00000400:00000010:2.0:1775767747.251001:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f990. 00000100:00000001:2.0:1775767747.251004:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767747.251007:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ff100 x1862022294230016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767772 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767747.251014:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767747.251017:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ff100 x1862022294230016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767772 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767747.251023:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767747.251025:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775767747.251028:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767747.251031:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:2.0:1775767747.251034:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767747.251036:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299672075 00000100:00000001:2.0:1775767747.251038:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299673406 00000100:00000001:2.0:1775767747.251040:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299672075 00000100:00000001:2.0:1775767747.251042:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299673406 00000100:00000001:2.0:1775767747.251044:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767747.251046:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767747.251048:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767747.251052:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767747/real 1775767747] req@ffff9942099ff100 x1862022294230016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775767747.251061:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767747.251062:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767747.251064:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b391b8. 00000400:00000200:2.0:1775767747.251066:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f770 00000400:00000010:2.0:1775767747.251068:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f770. 00000100:00000001:2.0:1775767747.251070:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767747.251073:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ff100 x1862022294230016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767747.251080:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ff100 x1862022294230016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767747.251086:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767747.251088:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767747.251090:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767747.251091:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767747.251094:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ff100 x1862022294230016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767747.251100:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767747.251103:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ff100 x1862022294230016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767772 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767747.251110:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767747.251112:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767747.251114:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767747.251115:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767747.251117:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767747.251119:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767747.251122:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767747.251125:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767747.251127:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767747.251132:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ff100 x1862022294230016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767772 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767747.251140:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294230016:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775767747.251144:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767747.251147:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fc000 x1862022294230144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767747.251153:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767747.251174:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767747.251177:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767747.251178:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767747.251181:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767747.251184:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294230144:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767747.251187:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767747.251189:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767747.251190:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767747.251191:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775767747.251194:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1e800. 02000000:00000001:2.0:1775767747.251196:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767747.251199:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767747.251201:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b391b8. 00000400:00000010:2.0:1775767747.251203:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f770. 00000100:00000200:2.0:1775767747.251206:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294230144, portal 4 00000100:00000001:2.0:1775767747.251207:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767747.251208:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641196544 : -112966068355072 : ffff9942099fc000) 00000100:00000040:2.0:1775767747.251213:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fc000 x1862022294230144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767772 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767747.251218:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767747.251219:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767747.251221:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f990. 00000100:00000200:2.0:1775767747.251222:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294230144, offset 0 00000400:00000200:2.0:1775767747.251226:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767747.251228:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767747.251231:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767747.251233:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767747.251235:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767747.251237:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767747.251238:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767747.251241:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224971:224971:256:4294967295] 192.168.202.156@tcp LPNI seq info [224967:224967:8:4294967295] 00000400:00000200:2.0:1775767747.251244:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767747.251248:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767747.251251:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767747.251253:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3a00. 00000800:00000200:2.0:1775767747.251255:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775767747.251257:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775767747.251260:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767747.251262:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000100:2.0:1775767747.251265:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767747.251266:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3a00. 00000400:00000200:2.0:1775767747.251269:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775767747.251271:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775767747.251273:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f990 00000400:00000010:2.0:1775767747.251274:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f990. 00000100:00000001:2.0:1775767747.251276:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767747.251278:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fc000 x1862022294230144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767772 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767747.251283:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767747.251285:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fc000 x1862022294230144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767772 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767747.251290:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767747.251291:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775767747.251293:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767747.251295:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:2.0:1775767747.251297:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767747.251299:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299672099 00000100:00000001:2.0:1775767747.251301:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299673406 00000100:00000001:2.0:1775767747.251303:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299672099 00000100:00000001:2.0:1775767747.251304:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299673406 00000100:00000001:2.0:1775767747.251305:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767747.251306:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767747.251308:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767747.251310:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767747/real 1775767747] req@ffff9942099fc000 x1862022294230144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775767747.251314:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767747.251315:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767747.251316:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b391b8. 00000400:00000200:2.0:1775767747.251318:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f770 00000400:00000010:2.0:1775767747.251319:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f770. 00000100:00000001:2.0:1775767747.251320:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767747.251322:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fc000 x1862022294230144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767747.251327:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fc000 x1862022294230144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767747.251331:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767747.251331:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767747.251332:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767747.251333:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767747.251335:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fc000 x1862022294230144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767747.251357:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767747.251361:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fc000 x1862022294230144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767772 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767747.251368:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767747.251371:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767747.251373:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767747.251374:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767747.251375:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767747.251380:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767747.251382:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767747.251386:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767747.251388:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767747.251391:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fc000 x1862022294230144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767772 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767747.251396:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294230144:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767747.251399:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767747.251400:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767747.251402:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fc700 x1862022294229888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767772 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767747.251406:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767747.251407:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767747.251409:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f400. 02000000:00000001:2.0:1775767747.251410:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767747.251411:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767747.251412:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:2.0:1775767747.251414:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767747.251415:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028000. 02000000:00000001:2.0:1775767747.251417:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767747.251418:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767747.251419:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fc700. 00000100:00000001:2.0:1775767747.251421:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767747.251422:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767747.251423:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767747.251425:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ff100 x1862022294230016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767772 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767747.251434:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767747.251435:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767747.251436:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f000. 02000000:00000001:2.0:1775767747.251438:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767747.251438:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767747.251439:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:2.0:1775767747.251440:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767747.251441:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029400. 02000000:00000001:2.0:1775767747.251442:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767747.251443:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767747.251444:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ff100. 00000100:00000001:2.0:1775767747.251446:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767747.251446:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767747.251447:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767747.251449:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fc000 x1862022294230144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767772 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767747.251452:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767747.251453:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767747.251454:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1e800. 02000000:00000001:2.0:1775767747.251456:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767747.251457:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767747.251458:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:2.0:1775767747.251459:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767747.251460:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002bc00. 02000000:00000001:2.0:1775767747.251461:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767747.251461:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767747.251462:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fc000. 00000100:00000001:2.0:1775767747.251463:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767747.251464:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767747.251465:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767747.251468:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767747.251468:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767747.251470:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775767747.251471:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767747.251471:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767747.251474:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ff800 x1862022294230272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767747.251479:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767747.251480:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767747.251482:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767747.251483:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767747.251484:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767747.251486:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294230272:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767747.251488:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767747.251489:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767747.251490:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767747.251492:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775767747.251494:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1e800. 02000000:00000001:2.0:1775767747.251495:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767747.251498:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767747.251499:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b391b8. 00000400:00000010:2.0:1775767747.251501:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f770. 00000100:00000200:2.0:1775767747.251504:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294230272, portal 4 00000100:00000001:2.0:1775767747.251505:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767747.251506:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641210880 : -112966068340736 : ffff9942099ff800) 00000100:00000040:2.0:1775767747.251510:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ff800 x1862022294230272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767772 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767747.251515:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767747.251516:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767747.251518:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f990. 00000100:00000200:2.0:1775767747.251520:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294230272, offset 0 00000400:00000200:2.0:1775767747.251522:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767747.251524:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767747.251526:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767747.251529:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767747.251531:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767747.251533:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767747.251534:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767747.251537:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224972:224972:256:4294967295] 192.168.202.156@tcp LPNI seq info [224968:224968:8:4294967295] 00000400:00000200:2.0:1775767747.251540:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767747.251544:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767747.251547:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767747.251550:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3a00. 00000800:00000200:2.0:1775767747.251552:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775767747.251554:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775767747.251557:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767747.251560:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000100:2.0:1775767747.251563:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767747.251565:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3a00. 00000400:00000200:2.0:1775767747.251567:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775767747.251569:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775767747.251571:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f990 00000400:00000010:2.0:1775767747.251572:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f990. 00000100:00000001:2.0:1775767747.251573:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767747.251576:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ff800 x1862022294230272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767772 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767747.251581:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767747.251583:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ff800 x1862022294230272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767772 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767747.251587:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767747.251588:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775767747.251590:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767747.251592:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:2.0:1775767747.251593:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767747.251595:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299672102 00000100:00000001:2.0:1775767747.251596:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299673406 00000100:00000001:2.0:1775767747.251598:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299672102 00000100:00000001:2.0:1775767747.251599:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299673406 00000100:00000001:2.0:1775767747.251601:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767747.251603:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767747.251604:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767747.251606:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767747/real 1775767747] req@ffff9942099ff800 x1862022294230272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775767747.251612:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767747.251613:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767747.251614:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b391b8. 00000400:00000200:2.0:1775767747.251615:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f770 00000400:00000010:2.0:1775767747.251616:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f770. 00000100:00000001:2.0:1775767747.251617:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767747.251619:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ff800 x1862022294230272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767747.251625:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ff800 x1862022294230272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767747.251628:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767747.251629:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767747.251630:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767747.251631:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767747.251633:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ff800 x1862022294230272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767747.251637:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767747.251639:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ff800 x1862022294230272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767772 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767747.251643:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767747.251644:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767747.251645:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767747.251646:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767747.251647:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767747.251650:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767747.251651:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767747.251655:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767747.251657:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767747.251662:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ff800 x1862022294230272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767772 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767747.251669:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294230272:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767747.251673:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767747.251675:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767747.251678:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ff800 x1862022294230272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767772 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767747.251684:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767747.251686:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767747.251687:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1e800. 02000000:00000001:2.0:1775767747.251690:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767747.251691:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767747.251693:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:2.0:1775767747.251695:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767747.251697:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002b800. 02000000:00000001:2.0:1775767747.251699:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767747.251700:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767747.251701:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ff800. 00000100:00000001:2.0:1775767747.251703:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767747.251705:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767747.251706:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767747.251709:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767747.251711:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767747.251713:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775767747.251714:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767748.273266:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767748.273273:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767748.273275:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767748.273280:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767748.273284:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767748.273287:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767748.273290:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767748.273292:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767748.273295:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767748.273363:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767748.273369:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767748.273372:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767748.273377:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767748.273380:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767748.273383:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767748.273386:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767748.273388:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767748.273391:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767748.337360:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767748.337364:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767748.337369:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775767748.593301:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299673406, at 4299673408 00000100:00000010:1.0:1775767748.593340:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d6660. 00000100:00000001:3.0:1775767749.297219:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767749.297224:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767749.297226:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767749.297232:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767749.297235:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767749.297238:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767749.297241:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767749.297243:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767749.297245:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767749.297315:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767749.297332:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767749.297334:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767749.297339:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775767749.297345:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767749.297349:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767749.297353:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767749.297355:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767749.297357:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767750.321244:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767750.321247:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767750.321253:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767750.321255:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767750.321256:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767750.321259:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767750.321265:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775767750.321266:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767750.321273:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767750.321273:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767750.321280:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767750.321280:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767750.321295:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767750.321297:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767750.321300:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767750.321305:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767750.321320:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767750.321324:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767750.385387:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767750.385391:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767750.385395:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775767751.345230:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767751.345230:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767751.345237:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767751.345238:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767751.345239:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767751.345240:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767751.345247:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775767751.345247:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767751.345252:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767751.345253:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767751.345255:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767751.345256:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767751.345258:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767751.345261:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767751.345263:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767751.345274:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767751.345276:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767751.345278:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767752.369223:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767752.369230:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767752.369233:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767752.369235:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767752.369239:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:3.0:1775767752.369241:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767752.369242:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767752.369246:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767752.369248:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767752.369249:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767752.369252:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767752.369253:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767752.369256:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767752.369260:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767752.369264:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767752.369265:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767752.369268:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767752.369273:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767752.369285:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767752.369302:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767752.369305:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775767752.369309:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775767752.369310:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767752.369316:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767752.369320:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1775767752.369322:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775767752.369327:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775767752.369333:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775767752.369334:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767752.369337:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767752.369339:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775767752.369344:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775767752.369348:0:13471:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5004 00000400:00000200:2.0:1775767752.369359:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767752.369367:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775767752.369371:0:13471:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775767752.369383:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767752.369388:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767752.369392:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767752.369396:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775767752.369400:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775767752.369403:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:2.0:1775767752.369406:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775767752.369410:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:2.0:1775767752.369413:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:2.0:1775767752.369416:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:2.0:1775767752.369423:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767752.369428:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767752.369432:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767752.369435:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775767752.369438:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:2.0:1775767752.369445:0:13471:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775767752.369452:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767752.369455:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775767752.369458:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767752.369462:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767752.369467:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775767752.369472:0:13471:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775767752.369479:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dff480. 00000020:00000040:2.0:1775767752.369486:0:13471:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:2.0:1775767752.369494:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775767752.369498:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767752.369502:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775767752.369505:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775767752.369518:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775767752.369530:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1e800. 00000100:00000001:2.0:1775767752.369537:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767752.369551:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dff480 x1862022294230400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775767752.369572:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775767752.369580:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767752.369588:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093a15639] to pc [ptlrpcd_rcv+-1] req@ffff994207dff480 x1862022294230400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767752.369615:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775767752.369621:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767752.369628:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775767752.369631:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775767752.369633:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775767752.369636:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775767752.369640:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5004 00000400:00000200:2.0:1775767752.369647:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767752.369649:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:2.0:1775767752.369653:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:3.0:1775767752.369655:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1775767752.369659:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1775767752.369660:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767752.369664:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767752.369667:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767752.369672:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dff480 x1862022294230400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775767752.369673:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767752.369677:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775767752.369683:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775767752.369686:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:2.0:1775767752.369689:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775767752.369693:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:2.0:1775767752.369697:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000001:3.0:1775767752.369697:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767752.369701:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767752.369702:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 02000000:00000001:3.0:1775767752.369707:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767752.369711:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1775767752.369716:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 02000000:00000001:3.0:1775767752.369716:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767752.369722:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:3.0:1775767752.369727:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294230400:192.168.202.156@tcp:250:kworker.0 00000100:00000040:2.0:1775767752.369728:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767752.369734:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:3.0:1775767752.369737:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000040:2.0:1775767752.369740:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 02000000:00000001:3.0:1775767752.369743:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767752.369747:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775767752.369748:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 02000000:00000001:3.0:1775767752.369754:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1775767752.369755:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767752.369760:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000010:3.0:1775767752.369761:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b2000. 02000000:00000001:3.0:1775767752.369767:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767752.369768:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767752.369775:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767752.369779:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767752.369781:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000010:3.0:1775767752.369785:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000008:00100000:2.0:1775767752.369788:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000400:00000010:3.0:1775767752.369796:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000008:00000001:2.0:1775767752.369804:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1775767752.369805:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294230400, portal 25 00000100:00000001:2.0:1775767752.369811:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767752.369811:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767752.369815:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611849856 : -112966097701760 : ffff994207dff480) 00000100:00000010:2.0:1775767752.369817:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfce00. 00000020:00000040:2.0:1775767752.369821:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000100:00000040:3.0:1775767752.369824:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dff480 x1862022294230400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767777 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1775767752.369825:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775767752.369828:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767752.369832:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775767752.369835:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775767752.369840:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767752.369846:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000010:2.0:1775767752.369849:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1f000. 00000100:00000040:3.0:1775767752.369853:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000100:00000001:2.0:1775767752.369857:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1775767752.369858:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767752.369866:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294230400, offset 0 00000100:00100000:2.0:1775767752.369870:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfce00 x1862022294230528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:3.0:1775767752.369880:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767752.369896:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767752.369898:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775767752.369902:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767752.369907:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00000040:2.0:1775767752.369910:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e5a7debe] to pc [ptlrpcd_rcv+-1] req@ffff994207dfce00 x1862022294230528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1775767752.369914:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767752.369925:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767752.369933:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767752.369941:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767752.369952:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224973:224973:256:4294967295] 192.168.202.156@tcp LPNI seq info [224969:224969:8:4294967295] 00000100:00000001:2.0:1775767752.369953:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767752.369962:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00080000:2.0:1775767752.369969:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767752.369975:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775767752.369979:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775767752.369982:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775767752.369989:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:3.0:1775767752.369994:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000100:00080000:2.0:1775767752.369998:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5004 00000400:00000200:2.0:1775767752.370009:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000800:00000200:3.0:1775767752.370011:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767752.370019:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000800:00000010:3.0:1775767752.370020:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000100:00080000:2.0:1775767752.370026:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000800:00000200:3.0:1775767752.370033:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000040:2.0:1775767752.370036:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000800:00000200:3.0:1775767752.370039:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000100:00000001:2.0:1775767752.370042:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767752.370049:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000800:00000200:3.1:1775767752.370053:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:2.0:1775767752.370055:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775767752.370060:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000800:00000200:3.1:1775767752.370060:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000020:00000040:2.0:1775767752.370064:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:2.0:1775767752.370068:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000800:00000100:3.0:1775767752.370072:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000020:00000040:2.0:1775767752.370073:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000800:00000010:3.0:1775767752.370077:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767752.370084:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000001:2.0:1775767752.370091:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000400:00000200:3.0:1775767752.370097:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000020:00000001:2.0:1775767752.370098:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000400:00000200:3.0:1775767752.370104:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000100:00000040:2.0:1775767752.370108:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000010:3.0:1775767752.370109:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:2.0:1775767752.370114:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1775767752.370116:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:2.0:1775767752.370121:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000200:3.0:1775767752.370126:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dff480 x1862022294230400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767777 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767752.370127:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775767752.370133:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:2.0:1775767752.370141:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775767752.370147:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767752.370152:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775767752.370185:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767752.370187:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1775767752.370190:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767752.370193:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775767752.370196:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00000040:3.0:1775767752.370196:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dff480 x1862022294230400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767777 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:2.0:1775767752.370202:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767752.370207:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775767752.370215:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfed80. 00000100:00000001:3.0:1775767752.370219:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775767752.370221:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000100:00000001:3.0:1775767752.370223:0:2399:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:2.0:1775767752.370227:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775767752.370230:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767752.370233:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1775767752.370234:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767752.370236:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775767752.370240:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767752.370242:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767752.370249:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299678526 02000000:00000010:2.0:1775767752.370250:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1f400. 00000100:00000010:3.0:1775767752.370258:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942079810c0. 00000100:00000001:2.0:1775767752.370259:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767752.370268:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00100000:2.0:1775767752.370271:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfed80 x1862022294230656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767752.370277:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299678526 00000100:00000010:3.0:1775767752.370285:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942079816c0. 00000100:00000001:3.0:1775767752.370305:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1775767752.370309:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767752.370312:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767752.370316:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767752.370318:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767752.370326:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb9b9eae] to pc [ptlrpcd_rcv+-1] req@ffff994207dfed80 x1862022294230656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767752.370334:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767752.370343:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767752/real 1775767752] req@ffff994207dff480 x1862022294230400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767752.370359:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775767752.370366:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767752.370375:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000400:00000001:3.0:1775767752.370376:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1775767752.370380:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000400:00000001:3.0:1775767752.370380:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:2.0:1775767752.370384:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000010:3.0:1775767752.370385:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000100:00000001:2.0:1775767752.370390:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:3.0:1775767752.370392:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000100:00080000:2.0:1775767752.370395:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5004 00000400:00000010:3.0:1775767752.370397:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767752.370404:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000200:2.0:1775767752.370405:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767752.370410:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775767752.370413:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000200:3.0:1775767752.370419:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dff480 x1862022294230400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775767752.370421:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767752.370427:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767752.370434:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767752.370441:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775767752.370445:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775767752.370449:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000100:00000200:3.0:1775767752.370449:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dff480 x1862022294230400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1775767752.370454:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775767752.370458:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:2.0:1775767752.370466:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:3.0:1775767752.370468:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:2.0:1775767752.370472:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:3.0:1775767752.370473:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767752.370479:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:2.0:1775767752.370481:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767752.370483:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767752.370488:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767752.370494:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00100000:3.0:1775767752.370494:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dff480 x1862022294230400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767752.370502:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775767752.370509:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:2.0:1775767752.370516:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775767752.370523:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767752.370530:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1775767752.370531:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1775767752.370536:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1775767752.370540:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dff480 x1862022294230400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767777 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1775767752.370544:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767752.370551:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775767752.370556:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:2.0:1775767752.370566:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767752.370572:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767752.370572:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767752.370578:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:2.0:1775767752.370580:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfe300. 00000100:00000001:3.0:1775767752.370582:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000040:2.0:1775767752.370586:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00000100:00000001:3.0:1775767752.370586:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1775767752.370593:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1775767752.370593:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 02000000:00000001:2.0:1775767752.370596:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767752.370596:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767752.370602:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:2.0:1775767752.370603:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775767752.370606:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775767752.370610:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767752.370611:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767752.370617:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000010:2.0:1775767752.370618:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1c800. 00000100:00000040:3.0:1775767752.370624:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dff480 x1862022294230400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767777 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767752.370625:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767752.370636:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfe300 x1862022294230784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767752.370642:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294230400:192.168.202.156@tcp:250:kworker.0 00000100:00000001:3.0:1775767752.370652:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767752.370657:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1775767752.370658:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775767752.370662:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767752.370663:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dff480 x1862022294230400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767777 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775767752.370669:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c661fef] to pc [ptlrpcd_rcv+-1] req@ffff994207dfe300 x1862022294230784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767752.370689:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767752.370693:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767752.370697:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b2000. 00000100:00000001:2.0:1775767752.370698:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767752.370701:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00080000:2.0:1775767752.370702:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000001:3.0:1775767752.370705:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767752.370708:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000100:00080000:2.0:1775767752.370711:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000020:00000001:3.0:1775767752.370712:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767752.370715:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1e800. 00000100:00000001:2.0:1775767752.370716:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775767752.370720:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:3.0:1775767752.370721:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767752.370725:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1775767752.370727:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000010:3.0:1775767752.370727:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dff480. 00000100:00080000:2.0:1775767752.370732:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5004 00000100:00000001:3.0:1775767752.370732:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767752.370735:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767752.370740:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1775767752.370746:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767752.370748:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767752.370751:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767752.370754:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:2.0:1775767752.370757:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:3.0:1775767752.370757:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767752.370761:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00080000:2.0:1775767752.370763:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767752.370767:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfce00 x1862022294230528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775767752.370772:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767752.370781:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1775767752.370787:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000040:2.0:1775767752.370788:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767752.370790:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767752.370795:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 02000000:00000001:3.0:1775767752.370795:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767752.370796:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1775767752.370799:0:13471:0:(genops.c:811:class_conn2export()) Process entered 02000000:00000001:3.0:1775767752.370800:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775767752.370803:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:2.0:1775767752.370807:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00100000:3.0:1775767752.370807:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294230528:192.168.202.156@tcp:38:kworker.0 00000020:00000040:2.0:1775767752.370810:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000100:00000001:3.0:1775767752.370813:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767752.370818:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767752.370821:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767752.370824:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 02000000:00000001:3.0:1775767752.370825:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767752.370829:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b2000. 00000020:00000001:2.0:1775767752.370834:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 02000000:00000001:3.0:1775767752.370834:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767752.370841:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000040:2.0:1775767752.370845:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000010:3.0:1775767752.370846:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000100:00000001:2.0:1775767752.370852:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:3.0:1775767752.370853:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767752.370858:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294230528, portal 10 00000100:00000040:2.0:1775767752.370859:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767752.370862:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767752.370865:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611840000 : -112966097711616 : ffff994207dfce00) 00000100:00000001:2.0:1775767752.370866:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775767752.370872:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00000040:3.0:1775767752.370873:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfce00 x1862022294230528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767777 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767752.370882:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775767752.370891:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767752.370892:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:2.0:1775767752.370896:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:3.0:1775767752.370899:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767752.370902:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 02000000:00000001:2.0:1775767752.370903:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000200:3.0:1775767752.370907:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294230528, offset 0 02000000:00000001:2.0:1775767752.370908:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767752.370914:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:3.0:1775767752.370916:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000008:00100000:2.0:1775767752.370918:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:2.0:1775767752.370922:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767752.370925:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767752.370928:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767752.370933:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00000010:2.0:1775767752.370936:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfc700. 00000400:00000200:3.0:1775767752.370939:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000040:2.0:1775767752.370941:0:13471:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00000400:00000200:3.0:1775767752.370945:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 02000000:00000001:2.0:1775767752.370950:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000200:3.0:1775767752.370950:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 02000000:00000001:2.0:1775767752.370954:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767752.370955:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00000001:2.0:1775767752.370961:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:3.0:1775767752.370963:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224974:224974:256:4294967295] 192.168.202.156@tcp LPNI seq info [224970:224970:8:4294967295] 02000000:00000001:2.0:1775767752.370965:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775767752.370970:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767752.370974:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000010:2.0:1775767752.370979:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1fc00. 00000400:00000200:3.0:1775767752.370986:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000100:00000001:2.0:1775767752.370988:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767752.370999:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfc700 x1862022294230912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:3.0:1775767752.371007:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00000001:2.0:1775767752.371017:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000010:3.0:1775767752.371019:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000100:00000001:2.0:1775767752.371022:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767752.371031:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a176dd1] to pc [ptlrpcd_rcv+-1] req@ffff994207dfc700 x1862022294230912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1775767752.371032:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767752.371040:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775767752.371055:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:2.0:1775767752.371056:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1775767752.371062:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000100:00000040:2.0:1775767752.371065:0:13471:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5014) 00000800:00000100:3.0:1775767752.371072:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767752.371077:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767752.371083:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767752.371091:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767752.371096:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767752.371100:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767752.371105:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767752.371110:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfce00 x1862022294230528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767777 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767752.371127:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767752.371132:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfce00 x1862022294230528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767777 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767752.371188:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767752.371193:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767752.371200:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767752.371206:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:3.0:1775767752.371214:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767752.371218:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299677204 00000100:00000001:3.0:1775767752.371221:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299678526 00000100:00000001:3.0:1775767752.371222:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299677204 00000100:00000001:3.0:1775767752.371224:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299678526 00000100:00000001:3.0:1775767752.371226:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767752.371231:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767752.371232:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767752.371236:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767752/real 1775767752] req@ffff994207dfce00 x1862022294230528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767752.371246:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767752.371249:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767752.371251:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767752.371254:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767752.371257:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767752.371262:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767752.371265:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfce00 x1862022294230528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767752.371279:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfce00 x1862022294230528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767752.371305:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767752.371308:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767752.371313:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767752.371316:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767752.371322:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfce00 x1862022294230528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767752.371334:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767752.371339:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfce00 x1862022294230528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767777 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767752.371351:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767752.371354:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767752.371358:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767752.371360:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767752.371364:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767752.371366:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767752.371373:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767752.371378:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767752.371381:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767752.371388:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfce00 x1862022294230528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767777 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767752.371402:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294230528:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767752.371410:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767752.371414:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfed80 x1862022294230656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767752.371432:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767752.371436:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767752.371440:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767752.371443:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767752.371446:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767752.371450:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294230656:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767752.371455:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767752.371458:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767752.371460:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767752.371462:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767752.371466:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b3400. 02000000:00000001:3.0:1775767752.371469:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767752.371474:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767752.371477:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767752.371480:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767752.371484:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294230656, portal 10 00000100:00000001:3.0:1775767752.371487:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767752.371490:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611848064 : -112966097703552 : ffff994207dfed80) 00000100:00000040:3.0:1775767752.371496:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfed80 x1862022294230656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767777 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767752.371507:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767752.371513:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767752.371515:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767752.371519:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294230656, offset 0 00000400:00000200:3.0:1775767752.371526:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767752.371534:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767752.371539:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767752.371545:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767752.371550:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767752.371555:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767752.371561:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767752.371569:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224975:224975:256:4294967295] 192.168.202.156@tcp LPNI seq info [224971:224971:8:4294967295] 00000400:00000200:3.0:1775767752.371577:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767752.371590:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767752.371598:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767752.371603:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767752.371609:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767752.371613:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775767752.371620:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767752.371623:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1775767752.371629:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767752.371631:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767752.371634:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767752.371639:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767752.371642:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767752.371644:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767752.371648:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767752.371652:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfed80 x1862022294230656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767777 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767752.371673:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767752.371676:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfed80 x1862022294230656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767777 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767752.371685:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767752.371688:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767752.371691:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767752.371694:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:3.0:1775767752.371700:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767752.371702:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299677196 00000100:00000001:3.0:1775767752.371705:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299678526 00000100:00000001:3.0:1775767752.371707:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299677196 00000100:00000001:3.0:1775767752.371711:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299678526 00000100:00000001:3.0:1775767752.371712:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767752.371716:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767752.371718:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767752.371723:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767752/real 1775767752] req@ffff994207dfed80 x1862022294230656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767752.371737:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767752.371739:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767752.371742:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767752.371745:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767752.371748:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767752.371751:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767752.371755:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfed80 x1862022294230656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767752.371770:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfed80 x1862022294230656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767752.371785:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767752.371789:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767752.371792:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767752.371794:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767752.371798:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfed80 x1862022294230656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767752.371807:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767752.371812:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfed80 x1862022294230656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767777 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767752.371823:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767752.371827:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767752.371829:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767752.371832:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767752.371834:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767752.371839:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767752.371841:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767752.371843:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767752.371845:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767752.371848:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfed80 x1862022294230656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767777 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767752.371854:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294230656:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767752.371857:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767752.371861:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfe300 x1862022294230784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767752.371865:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767752.371866:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767752.371867:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767752.371869:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767752.371870:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767752.371872:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294230784:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767752.371874:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767752.371875:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767752.371877:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767752.371878:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767752.371880:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b0400. 02000000:00000001:3.0:1775767752.371881:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767752.371883:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767752.371885:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767752.371887:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767752.371889:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294230784, portal 4 00000100:00000001:3.0:1775767752.371890:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767752.371891:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611845376 : -112966097706240 : ffff994207dfe300) 00000100:00000040:3.0:1775767752.371895:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfe300 x1862022294230784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767777 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767752.371899:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767752.371901:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767752.371902:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767752.371903:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294230784, offset 0 00000400:00000200:3.0:1775767752.371906:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767752.371908:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767752.371909:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767752.371911:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767752.371913:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767752.371914:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767752.371916:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767752.371918:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224976:224976:256:4294967295] 192.168.202.156@tcp LPNI seq info [224972:224972:8:4294967295] 00000400:00000200:3.0:1775767752.371921:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767752.371925:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767752.371928:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767752.371930:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767752.371932:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767752.371934:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775767752.371937:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767752.371938:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1775767752.371941:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767752.371942:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767752.371944:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767752.371946:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767752.371948:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767752.371949:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767752.371950:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767752.371952:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfe300 x1862022294230784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767777 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767752.371957:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767752.371959:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfe300 x1862022294230784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767777 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767752.371963:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767752.371964:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767752.371966:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767752.371968:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:3.0:1775767752.371970:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767752.371971:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299677219 00000100:00000001:3.0:1775767752.371973:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299678526 00000100:00000001:3.0:1775767752.371974:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299677219 00000100:00000001:3.0:1775767752.371976:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299678526 00000100:00000001:3.0:1775767752.371977:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767752.371978:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767752.371979:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767752.371981:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767752/real 1775767752] req@ffff994207dfe300 x1862022294230784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767752.371986:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767752.371987:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767752.371988:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767752.371989:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767752.371990:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767752.371991:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767752.371994:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfe300 x1862022294230784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767752.371998:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfe300 x1862022294230784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767752.372002:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767752.372003:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767752.372004:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767752.372005:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767752.372008:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfe300 x1862022294230784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767752.372015:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767752.372019:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfe300 x1862022294230784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767777 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767752.372026:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767752.372028:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767752.372030:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767752.372031:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767752.372033:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767752.372037:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767752.372039:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767752.372043:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767752.372045:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767752.372049:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfe300 x1862022294230784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767777 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767752.372065:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294230784:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767752.372069:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767752.372071:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767752.372073:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfce00 x1862022294230528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767777 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767752.372078:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767752.372079:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767752.372080:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b2000. 02000000:00000001:3.0:1775767752.372082:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767752.372083:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767752.372084:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:3.0:1775767752.372086:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767752.372087:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1f000. 02000000:00000001:3.0:1775767752.372089:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767752.372090:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767752.372091:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfce00. 00000100:00000001:3.0:1775767752.372093:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767752.372094:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767752.372095:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767752.372097:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfed80 x1862022294230656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767777 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767752.372102:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767752.372102:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767752.372103:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b3400. 02000000:00000001:3.0:1775767752.372105:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767752.372106:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767752.372106:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:3.0:1775767752.372107:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767752.372108:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1f400. 02000000:00000001:3.0:1775767752.372109:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767752.372110:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767752.372111:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfed80. 00000100:00000001:3.0:1775767752.372113:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767752.372114:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767752.372115:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767752.372117:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfe300 x1862022294230784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767777 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767752.372121:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767752.372122:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767752.372122:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b0400. 02000000:00000001:3.0:1775767752.372124:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767752.372125:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767752.372125:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:3.0:1775767752.372126:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767752.372127:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1c800. 02000000:00000001:3.0:1775767752.372128:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767752.372129:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767752.372130:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfe300. 00000100:00000001:3.0:1775767752.372131:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767752.372132:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767752.372133:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767752.372136:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767752.372137:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767752.372139:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767752.372140:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767752.372141:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767752.372144:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfc700 x1862022294230912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767752.372151:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767752.372152:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767752.372174:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767752.372176:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767752.372178:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767752.372181:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294230912:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767752.372184:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767752.372186:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767752.372188:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767752.372190:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767752.372194:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b0400. 02000000:00000001:3.0:1775767752.372196:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767752.372199:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767752.372202:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767752.372205:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767752.372208:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294230912, portal 4 00000100:00000001:3.0:1775767752.372210:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767752.372211:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611838208 : -112966097713408 : ffff994207dfc700) 00000100:00000040:3.0:1775767752.372217:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfc700 x1862022294230912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767777 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767752.372223:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767752.372226:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767752.372228:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767752.372231:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294230912, offset 0 00000400:00000200:3.0:1775767752.372235:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767752.372239:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767752.372242:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767752.372245:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767752.372248:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767752.372250:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767752.372253:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767752.372256:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224977:224977:256:4294967295] 192.168.202.156@tcp LPNI seq info [224973:224973:8:4294967295] 00000400:00000200:3.0:1775767752.372261:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767752.372267:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767752.372272:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767752.372275:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767752.372278:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767752.372280:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775767752.372283:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767752.372284:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1775767752.372287:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767752.372299:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767752.372302:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767752.372305:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767752.372307:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767752.372308:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767752.372310:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767752.372312:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfc700 x1862022294230912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767777 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767752.372317:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767752.372319:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfc700 x1862022294230912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767777 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767752.372324:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767752.372325:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767752.372327:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767752.372329:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:3.0:1775767752.372332:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767752.372333:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299677223 00000100:00000001:3.0:1775767752.372335:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299678526 00000100:00000001:3.0:1775767752.372336:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299677223 00000100:00000001:3.0:1775767752.372338:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299678526 00000100:00000001:3.0:1775767752.372339:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767752.372340:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767752.372341:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767752.372343:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767752/real 1775767752] req@ffff994207dfc700 x1862022294230912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767752.372348:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767752.372349:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767752.372350:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767752.372352:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767752.372353:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767752.372354:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767752.372357:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfc700 x1862022294230912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767752.372362:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfc700 x1862022294230912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767752.372366:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767752.372367:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767752.372368:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767752.372369:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767752.372371:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfc700 x1862022294230912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767752.372374:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767752.372377:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfc700 x1862022294230912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767777 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767752.372380:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767752.372382:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767752.372383:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767752.372384:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767752.372385:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767752.372387:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767752.372389:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767752.372391:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767752.372392:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767752.372394:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfc700 x1862022294230912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767777 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767752.372399:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294230912:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767752.372401:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767752.372403:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767752.372404:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfc700 x1862022294230912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767777 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767752.372409:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767752.372410:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767752.372411:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b0400. 02000000:00000001:3.0:1775767752.372413:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767752.372413:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767752.372414:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:3.0:1775767752.372415:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767752.372416:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1fc00. 02000000:00000001:3.0:1775767752.372418:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767752.372419:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767752.372420:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfc700. 00000100:00000001:3.0:1775767752.372421:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767752.372422:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767752.372423:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767752.372425:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767752.372426:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767752.372427:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767752.372429:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767752.433341:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767752.433344:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767752.433346:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775767753.393220:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767753.393220:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767753.393226:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767753.393227:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767753.393231:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767753.393231:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767753.393232:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767753.393237:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:2.0:1775767753.393238:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:1.0:1775767753.393239:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767753.393242:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767753.393244:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767753.393245:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1775767753.393246:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767753.393248:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767753.393251:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767753.393252:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767753.393254:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767753.393256:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767753.393257:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767753.393260:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767753.393260:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767753.393263:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767753.393266:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767753.393269:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767753.393272:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767753.393276:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767753.713285:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299678526, at 4299678528 00000100:00000010:1.0:1775767753.713304:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942079816c0. 00000100:00000001:1.0:1775767754.417222:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767754.417230:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767754.417232:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767754.417239:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775767754.417244:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767754.417245:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767754.417249:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767754.417251:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767754.417254:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767754.417255:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767754.417255:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767754.417260:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767754.417261:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767754.417261:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767754.417263:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767754.417267:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767754.417278:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767754.417283:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767754.417284:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767754.417287:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767754.417289:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767754.417292:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767754.417297:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767754.417301:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767754.417304:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767754.417308:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767754.417311:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767754.481367:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767754.481370:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767754.481374:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775767755.441272:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767755.441274:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767755.441281:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767755.441281:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767755.441285:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767755.441286:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767755.441294:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775767755.441294:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767755.441301:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767755.441301:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767755.441307:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767755.441307:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767755.441312:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767755.441315:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767755.441319:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767755.441319:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767755.441323:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767755.441327:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767755.441360:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767755.441364:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767755.441366:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767755.441369:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775767755.441372:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767755.441375:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767755.441377:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767755.441378:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767755.441381:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767756.465260:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767756.465262:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767756.465268:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767756.465269:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767756.465272:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767756.465273:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767756.465280:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:2.0:1775767756.465280:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775767756.465281:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767756.465284:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767756.465287:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767756.465287:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767756.465288:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1775767756.465290:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767756.465294:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767756.465294:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767756.465296:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767756.465299:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767756.465300:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767756.465302:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767756.465306:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767756.465306:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767756.465309:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767756.465312:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767756.465312:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767756.465316:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767756.465319:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767756.529380:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767756.529383:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767756.529386:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775767757.489269:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767757.489269:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767757.489271:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767757.489277:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767757.489277:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767757.489277:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767757.489281:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767757.489282:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767757.489282:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767757.489289:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775767757.489289:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:2.0:1775767757.489290:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767757.489296:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767757.489297:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767757.489297:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767757.489302:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767757.489303:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767757.489303:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767757.489308:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767757.489311:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767757.489315:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767757.489315:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767757.489317:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767757.489319:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767757.489321:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767757.489324:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767757.489328:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775767757.489373:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767757.489377:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775767757.489379:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775767757.489382:0:36:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775767757.489385:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775767757.489391:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5009 00000400:00000200:2.0:1775767757.489401:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767757.489407:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775767757.489410:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775767757.489413:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767757.489416:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767757.489419:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767757.489421:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775767757.489426:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775767757.489428:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:2.0:1775767757.489431:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775767757.489433:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:2.0:1775767757.489436:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:2.0:1775767757.489438:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:2.0:1775767757.489442:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767757.489444:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767757.489447:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767757.489449:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775767757.489452:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:2.0:1775767757.489455:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775767757.489458:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767757.489460:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775767757.489463:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767757.489465:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767757.489466:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775767757.489468:0:36:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775767757.489476:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dff100. 00000020:00000040:2.0:1775767757.489479:0:36:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:2.0:1775767757.489481:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775767757.489482:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767757.489484:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775767757.489486:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775767757.489490:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775767757.489496:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1d000. 00000100:00000001:2.0:1775767757.489501:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767757.489508:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dff100 x1862022294231040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775767757.489516:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775767757.489519:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767757.489532:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3e64ae1] to pc [ptlrpcd_rcv+-1] req@ffff994207dff100 x1862022294231040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767757.489546:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775767757.489548:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767757.489551:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775767757.489552:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775767757.489553:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775767757.489555:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775767757.489557:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5009 00000400:00000200:2.0:1775767757.489560:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767757.489563:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775767757.489565:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775767757.489568:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767757.489569:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767757.489572:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767757.489575:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:3.0:1775767757.489575:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1775767757.489579:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775767757.489581:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000100:00000001:3.0:1775767757.489581:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767757.489584:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000001:2.0:1775767757.489585:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775767757.489588:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:2.0:1775767757.489592:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:3.0:1775767757.489594:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dff100 x1862022294231040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1775767757.489595:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:2.0:1775767757.489603:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767757.489608:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767757.489611:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767757.489613:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775767757.489616:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00080000:2.0:1775767757.489621:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767757.489621:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767757.489624:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767757.489625:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767757.489629:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767757.489630:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767757.489634:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767757.489638:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767757.489639:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767757.489645:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767757.489649:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00100000:3.0:1775767757.489649:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294231040:192.168.202.156@tcp:250:kworker.0 00000008:00100000:2.0:1775767757.489654:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1775767757.489656:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000008:00000001:2.0:1775767757.489660:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767757.489661:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1775767757.489663:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767757.489665:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775767757.489668:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfdc00. 02000000:00000001:3.0:1775767757.489670:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:2.0:1775767757.489672:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:2.0:1775767757.489676:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000010:3.0:1775767757.489677:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b0400. 02000000:00000001:2.0:1775767757.489678:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767757.489682:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767757.489682:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767757.489685:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775767757.489687:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775767757.489692:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1e400. 00000100:00000001:2.0:1775767757.489698:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767757.489698:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767757.489702:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000100:00100000:2.0:1775767757.489707:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfdc00 x1862022294231168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:3.0:1775767757.489710:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000001:2.0:1775767757.489716:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775767757.489718:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767757.489720:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003e25603c] to pc [ptlrpcd_rcv+-1] req@ffff994207dfdc00 x1862022294231168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767757.489720:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294231040, portal 25 00000100:00000001:3.0:1775767757.489726:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767757.489730:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611848960 : -112966097702656 : ffff994207dff100) 00000100:00000001:2.0:1775767757.489732:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775767757.489737:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767757.489740:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775767757.489743:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775767757.489744:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:3.0:1775767757.489744:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dff100 x1862022294231040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767782 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767757.489749:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775767757.489753:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5009 00000400:00000200:2.0:1775767757.489756:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767757.489758:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775767757.489761:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775767757.489767:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767757.489771:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767757.489775:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767757.489775:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1775767757.489776:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775767757.489779:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775767757.489781:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:2.0:1775767757.489783:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000040:3.0:1775767757.489784:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000020:00000040:2.0:1775767757.489785:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:2.0:1775767757.489788:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:2.0:1775767757.489790:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000400:00000010:3.0:1775767757.489790:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000040:2.0:1775767757.489796:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767757.489798:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:3.0:1775767757.489798:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294231040, offset 0 00000100:00000040:2.0:1775767757.489801:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767757.489804:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775767757.489805:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:2.0:1775767757.489810:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000400:00000200:3.0:1775767757.489812:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00000001:2.0:1775767757.489813:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767757.489816:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775767757.489819:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767757.489820:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767757.489821:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775767757.489823:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775767757.489828:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767757.489828:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767757.489831:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775767757.489836:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfc700. 00000020:00000040:2.0:1775767757.489839:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000400:00000200:3.0:1775767757.489839:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 02000000:00000001:2.0:1775767757.489844:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775767757.489845:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767757.489847:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1775767757.489848:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775767757.489851:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775767757.489852:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775767757.489855:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1f800. 00000100:00000001:2.0:1775767757.489859:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767757.489861:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000100:00100000:2.0:1775767757.489867:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfc700 x1862022294231296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:3.0:1775767757.489868:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767757.489873:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767757.489881:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224978:224978:256:4294967295] 192.168.202.156@tcp LPNI seq info [224974:224974:8:4294967295] 00000100:00000001:2.0:1775767757.489887:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775767757.489890:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767757.489893:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000040:2.0:1775767757.489898:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a176dd1] to pc [ptlrpcd_rcv+-1] req@ffff994207dfc700 x1862022294231296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767757.489915:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767757.489915:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000100:00080000:2.0:1775767757.489921:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:3.0:1775767757.489927:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00080000:2.0:1775767757.489928:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1775767757.489932:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775767757.489934:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000010:3.0:1775767757.489936:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000100:00000001:2.0:1775767757.489938:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775767757.489942:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5009 00000400:00000200:2.0:1775767757.489946:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767757.489951:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000800:00000200:3.0:1775767757.489951:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00080000:2.0:1775767757.489955:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775767757.489958:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000800:00000200:3.0:1775767757.489959:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000100:00000001:2.0:1775767757.489960:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767757.489964:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767757.489967:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775767757.489969:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775767757.489971:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:2.0:1775767757.489973:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000800:00000200:3.1:1775767757.489973:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000040:2.0:1775767757.489975:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:2.0:1775767757.489979:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:2.0:1775767757.489981:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000800:00000200:3.1:1775767757.489981:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000100:00000040:2.0:1775767757.489987:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767757.489989:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767757.489993:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767757.489994:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000800:00000100:3.0:1775767757.489994:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000020:00000040:2.0:1775767757.489998:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:2.0:1775767757.490000:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000800:00000010:3.0:1775767757.490000:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000100:00000001:2.0:1775767757.490005:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767757.490007:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775767757.490009:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1775767757.490010:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000001:2.0:1775767757.490011:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767757.490015:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775767757.490018:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:2.0:1775767757.490021:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767757.490022:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000100:00000001:2.0:1775767757.490023:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775767757.490027:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfe300. 00000020:00000040:2.0:1775767757.490030:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00000400:00000200:3.0:1775767757.490030:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 02000000:00000001:2.0:1775767757.490032:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775767757.490035:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1775767757.490036:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:2.0:1775767757.490037:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775767757.490039:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775767757.490041:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767757.490044:0:2399:0:(events.c:34:request_out_callback()) Process entered 02000000:00000010:2.0:1775767757.490045:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1d400. 00000100:00000001:2.0:1775767757.490049:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1775767757.490053:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dff100 x1862022294231040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767782 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767757.490056:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfe300 x1862022294231424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775767757.490083:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775767757.490086:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767757.490088:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767757.490089:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c661fef] to pc [ptlrpcd_rcv+-1] req@ffff994207dfe300 x1862022294231424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775767757.490097:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dff100 x1862022294231040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767782 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767757.490098:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775767757.490103:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767757.490110:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1775767757.490113:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775767757.490115:0:36:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775767757.490119:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:3.0:1775767757.490120:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775767757.490122:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5009 00000100:00000001:3.0:1775767757.490125:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:2.0:1775767757.490127:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767757.490130:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775767757.490131:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775767757.490134:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767757.490134:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767757.490136:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767757.490139:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767757.490140:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000200:3.0:1775767757.490143:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:2.0:1775767757.490144:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775767757.490146:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:2.0:1775767757.490148:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775767757.490149:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000100:00000001:3.0:1775767757.490149:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299683645 00000020:00000001:2.0:1775767757.490152:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:2.0:1775767757.490173:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:2.0:1775767757.490175:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767757.490176:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767757.490177:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767757.490179:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775767757.490180:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:2.0:1775767757.490182:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775767757.490184:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767757.490185:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775767757.490187:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767757.490189:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767757.490189:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942079811e0. 00000100:00000001:2.0:1775767757.490191:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775767757.490194:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:2.0:1775767757.490197:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767757.490199:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767757.490199:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000010:2.0:1775767757.490204:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfed80. 00000020:00000040:2.0:1775767757.490207:0:36:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:2.0:1775767757.490209:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1775767757.490209:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299683646 02000000:00000001:2.0:1775767757.490211:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767757.490214:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775767757.490215:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775767757.490216:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767757.490217:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942079816c0. 02000000:00000010:2.0:1775767757.490220:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1fc00. 00000100:00000001:3.0:1775767757.490222:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1775767757.490223:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767757.490230:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfed80 x1862022294231552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767757.490230:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767757.490237:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767757.490256:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767757.490256:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1775767757.490260:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767757.490266:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb9b9eae] to pc [ptlrpcd_rcv+-1] req@ffff994207dfed80 x1862022294231552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767757.490266:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767757/real 1775767757] req@ffff994207dff100 x1862022294231040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767757.490287:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767757.490293:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5020) 00000400:00000001:3.0:1775767757.490295:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767757.490299:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767757.490302:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767757.490306:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767757.490308:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767757.490311:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767757.490316:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dff100 x1862022294231040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767757.490327:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dff100 x1862022294231040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767757.490335:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767757.490336:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767757.490339:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767757.490341:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767757.490345:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dff100 x1862022294231040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767757.490353:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767757.490357:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dff100 x1862022294231040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767782 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767757.490365:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767757.490368:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767757.490370:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767757.490372:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767757.490374:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767757.490376:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767757.490379:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767757.490384:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767757.490396:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767757.490401:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dff100 x1862022294231040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767782 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767757.490410:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294231040:192.168.202.156@tcp:250:kworker.0 00000100:00000001:3.0:1775767757.490415:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767757.490418:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767757.490421:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dff100 x1862022294231040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767782 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767757.490429:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767757.490431:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767757.490434:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b0400. 02000000:00000001:3.0:1775767757.490438:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767757.490440:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767757.490441:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:3.0:1775767757.490443:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767757.490446:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1d000. 02000000:00000001:3.0:1775767757.490450:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767757.490452:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767757.490454:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dff100. 00000100:00000001:3.0:1775767757.490457:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767757.490459:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767757.490470:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767757.490477:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767757.490478:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767757.490480:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767757.490482:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767757.490484:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767757.490487:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfdc00 x1862022294231168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767757.490496:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767757.490498:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767757.490500:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767757.490502:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767757.490504:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767757.490508:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294231168:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767757.490511:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767757.490513:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767757.490515:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767757.490518:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767757.490521:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b0400. 02000000:00000001:3.0:1775767757.490523:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767757.490527:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767757.490530:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767757.490534:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767757.490538:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294231168, portal 10 00000100:00000001:3.0:1775767757.490540:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767757.490541:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611843584 : -112966097708032 : ffff994207dfdc00) 00000100:00000040:3.0:1775767757.490548:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfdc00 x1862022294231168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767782 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767757.490555:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767757.490558:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767757.490560:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767757.490563:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294231168, offset 0 00000400:00000200:3.0:1775767757.490568:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767757.490572:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767757.490576:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767757.490580:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767757.490582:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767757.490585:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767757.490588:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767757.490591:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224979:224979:256:4294967295] 192.168.202.156@tcp LPNI seq info [224975:224975:8:4294967295] 00000400:00000200:3.0:1775767757.490595:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767757.490603:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767757.490608:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767757.490612:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767757.490616:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767757.490619:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1775767757.490624:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767757.490627:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000100:3.0:1775767757.490631:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767757.490633:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767757.490637:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767757.490641:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767757.490644:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767757.490646:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767757.490648:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767757.490652:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfdc00 x1862022294231168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767782 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767757.490661:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767757.490664:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfdc00 x1862022294231168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767782 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767757.490671:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767757.490673:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767757.490677:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767757.490680:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:3.0:1775767757.490684:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767757.490686:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299682323 00000100:00000001:3.0:1775767757.490689:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299683645 00000100:00000001:3.0:1775767757.490692:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299682323 00000100:00000001:3.0:1775767757.490694:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299683646 00000100:00000001:3.0:1775767757.490697:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767757.490699:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767757.490701:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767757.490705:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767757/real 1775767757] req@ffff994207dfdc00 x1862022294231168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767757.490714:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767757.490715:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767757.490717:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767757.490719:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767757.490721:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767757.490723:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767757.490727:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfdc00 x1862022294231168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767757.490737:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfdc00 x1862022294231168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767757.490744:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767757.490746:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767757.490748:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767757.490749:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767757.490753:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfdc00 x1862022294231168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767757.490760:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767757.490764:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfdc00 x1862022294231168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767782 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767757.490772:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767757.490775:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767757.490776:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767757.490778:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767757.490780:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767757.490783:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767757.490785:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767757.490789:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767757.490791:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767757.490795:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfdc00 x1862022294231168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767782 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767757.490804:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294231168:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767757.490809:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767757.490812:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfc700 x1862022294231296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767757.490825:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767757.490827:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767757.490829:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767757.490831:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767757.490833:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767757.490836:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294231296:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767757.490840:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767757.490842:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767757.490843:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767757.490845:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767757.490849:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b3400. 02000000:00000001:3.0:1775767757.490851:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767757.490855:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767757.490858:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767757.490861:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767757.490864:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294231296, portal 10 00000100:00000001:3.0:1775767757.490866:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767757.490868:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611838208 : -112966097713408 : ffff994207dfc700) 00000100:00000040:3.0:1775767757.490874:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfc700 x1862022294231296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767782 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767757.490881:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767757.490883:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767757.490886:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767757.490889:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294231296, offset 0 00000400:00000200:3.0:1775767757.490893:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767757.490898:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767757.490902:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767757.490905:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767757.490908:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767757.490910:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767757.490913:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767757.490917:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224980:224980:256:4294967295] 192.168.202.156@tcp LPNI seq info [224976:224976:8:4294967295] 00000400:00000200:3.0:1775767757.490921:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767757.490928:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767757.490933:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767757.490937:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767757.490941:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767757.490944:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1775767757.490948:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767757.490951:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000100:3.0:1775767757.490955:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767757.490957:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767757.490960:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767757.490964:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767757.490967:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767757.490969:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767757.490971:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767757.490975:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfc700 x1862022294231296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767782 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767757.490983:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767757.490986:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfc700 x1862022294231296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767782 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767757.490993:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767757.490995:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767757.490998:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767757.491001:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:3.0:1775767757.491005:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767757.491007:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299682315 00000100:00000001:3.0:1775767757.491010:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299683645 00000100:00000001:3.0:1775767757.491012:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299682315 00000100:00000001:3.0:1775767757.491014:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299683646 00000100:00000001:3.0:1775767757.491017:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767757.491019:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767757.491021:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767757.491025:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767757/real 1775767757] req@ffff994207dfc700 x1862022294231296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767757.491034:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767757.491036:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767757.491037:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767757.491040:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767757.491041:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767757.491044:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767757.491047:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfc700 x1862022294231296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767757.491056:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfc700 x1862022294231296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767757.491063:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767757.491065:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767757.491067:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767757.491068:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767757.491072:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfc700 x1862022294231296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767757.491080:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767757.491084:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfc700 x1862022294231296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767782 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767757.491092:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767757.491094:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767757.491096:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767757.491098:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767757.491099:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767757.491102:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767757.491105:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767757.491109:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767757.491112:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767757.491116:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfc700 x1862022294231296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767782 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767757.491126:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294231296:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767757.491130:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767757.491133:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfe300 x1862022294231424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767757.491140:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767757.491142:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767757.491144:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767757.491146:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767757.491148:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767757.491151:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294231424:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767757.491174:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767757.491176:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767757.491178:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767757.491180:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767757.491183:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b2000. 02000000:00000001:3.0:1775767757.491186:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767757.491190:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767757.491192:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767757.491196:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767757.491199:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294231424, portal 4 00000100:00000001:3.0:1775767757.491201:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767757.491202:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611845376 : -112966097706240 : ffff994207dfe300) 00000100:00000040:3.0:1775767757.491208:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfe300 x1862022294231424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767782 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767757.491215:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767757.491218:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767757.491220:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767757.491222:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294231424, offset 0 00000400:00000200:3.0:1775767757.491227:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767757.491231:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767757.491235:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767757.491252:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767757.491255:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767757.491258:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767757.491261:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767757.491264:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224981:224981:256:4294967295] 192.168.202.156@tcp LPNI seq info [224977:224977:8:4294967295] 00000400:00000200:3.0:1775767757.491268:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767757.491275:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767757.491280:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767757.491284:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767757.491288:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767757.491291:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1775767757.491295:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767757.491298:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000100:3.0:1775767757.491302:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767757.491304:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767757.491307:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767757.491311:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767757.491314:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767757.491316:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767757.491318:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767757.491322:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfe300 x1862022294231424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767782 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767757.491330:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767757.491334:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfe300 x1862022294231424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767782 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767757.491340:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767757.491342:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767757.491346:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767757.491349:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:3.0:1775767757.491352:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767757.491355:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299682339 00000100:00000001:3.0:1775767757.491357:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299683645 00000100:00000001:3.0:1775767757.491359:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299682339 00000100:00000001:3.0:1775767757.491362:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299683646 00000100:00000001:3.0:1775767757.491364:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767757.491366:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767757.491368:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767757.491372:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767757/real 1775767757] req@ffff994207dfe300 x1862022294231424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767757.491380:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767757.491382:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767757.491383:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767757.491386:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767757.491388:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767757.491390:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767757.491393:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfe300 x1862022294231424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767757.491403:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfe300 x1862022294231424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767757.491409:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767757.491411:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767757.491413:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767757.491414:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767757.491418:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfe300 x1862022294231424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767757.491425:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767757.491428:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfe300 x1862022294231424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767782 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767757.491436:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767757.491440:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767757.491441:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767757.491443:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767757.491445:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767757.491449:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767757.491451:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767757.491455:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767757.491457:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767757.491461:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfe300 x1862022294231424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767782 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767757.491470:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294231424:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767757.491474:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767757.491477:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfed80 x1862022294231552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767757.491484:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767757.491485:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767757.491488:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767757.491489:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767757.491492:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767757.491494:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294231552:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767757.491498:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767757.491500:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767757.491501:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767757.491503:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767757.491507:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b1000. 02000000:00000001:3.0:1775767757.491509:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767757.491512:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767757.491515:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767757.491518:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767757.491521:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294231552, portal 4 00000100:00000001:3.0:1775767757.491523:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767757.491525:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611848064 : -112966097703552 : ffff994207dfed80) 00000100:00000040:3.0:1775767757.491531:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfed80 x1862022294231552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767782 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767757.491538:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767757.491540:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767757.491543:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767757.491545:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294231552, offset 0 00000400:00000200:3.0:1775767757.491549:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767757.491553:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767757.491557:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767757.491560:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767757.491563:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767757.491565:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767757.491567:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767757.491571:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224982:224982:256:4294967295] 192.168.202.156@tcp LPNI seq info [224978:224978:8:4294967295] 00000400:00000200:3.0:1775767757.491575:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767757.491582:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767757.491587:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767757.491590:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767757.491594:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767757.491597:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1775767757.491602:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767757.491604:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000100:3.0:1775767757.491608:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767757.491610:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767757.491613:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767757.491617:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767757.491620:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767757.491622:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767757.491625:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767757.491629:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfed80 x1862022294231552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767782 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767757.491637:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767757.491640:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfed80 x1862022294231552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767782 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767757.491647:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767757.491649:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767757.491652:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767757.491655:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:3.0:1775767757.491658:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767757.491660:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299682342 00000100:00000001:3.0:1775767757.491663:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299683645 00000100:00000001:3.0:1775767757.491665:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299682342 00000100:00000001:3.0:1775767757.491667:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299683646 00000100:00000001:3.0:1775767757.491669:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767757.491672:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767757.491674:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767757.491678:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767757/real 1775767757] req@ffff994207dfed80 x1862022294231552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767757.491687:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767757.491688:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767757.491690:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767757.491692:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767757.491694:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767757.491696:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767757.491700:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfed80 x1862022294231552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767757.491709:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfed80 x1862022294231552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767757.491722:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767757.491723:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767757.491725:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767757.491727:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767757.491731:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfed80 x1862022294231552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767757.491739:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767757.491743:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfed80 x1862022294231552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767782 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767757.491751:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767757.491753:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767757.491755:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767757.491757:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767757.491759:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767757.491762:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767757.491765:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767757.491768:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767757.491770:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767757.491774:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfed80 x1862022294231552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767782 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767757.491783:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294231552:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767757.491787:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767757.491789:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767757.491792:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfdc00 x1862022294231168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767782 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767757.491798:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767757.491800:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767757.491802:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b0400. 02000000:00000001:3.0:1775767757.491805:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767757.491806:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767757.491808:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:3.0:1775767757.491810:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767757.491812:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1e400. 02000000:00000001:3.0:1775767757.491815:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767757.491816:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767757.491818:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfdc00. 00000100:00000001:3.0:1775767757.491820:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767757.491822:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767757.491823:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767757.491827:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfc700 x1862022294231296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767782 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767757.491834:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767757.491835:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767757.491837:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b3400. 02000000:00000001:3.0:1775767757.491840:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767757.491841:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767757.491842:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:3.0:1775767757.491844:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767757.491846:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1f800. 02000000:00000001:3.0:1775767757.491848:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767757.491849:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767757.491851:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfc700. 00000100:00000001:3.0:1775767757.491853:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767757.491855:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767757.491856:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767757.491860:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfe300 x1862022294231424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767782 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767757.491867:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767757.491869:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767757.491870:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b2000. 02000000:00000001:3.0:1775767757.491872:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767757.491874:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767757.491875:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:3.0:1775767757.491877:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767757.491878:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1d400. 02000000:00000001:3.0:1775767757.491881:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767757.491882:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767757.491883:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfe300. 00000100:00000001:3.0:1775767757.491886:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767757.491887:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767757.491889:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767757.491893:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfed80 x1862022294231552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767782 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767757.491900:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767757.491901:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767757.491903:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b1000. 02000000:00000001:3.0:1775767757.491905:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767757.491906:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767757.491907:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:3.0:1775767757.491909:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767757.491911:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1fc00. 02000000:00000001:3.0:1775767757.491913:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767757.491914:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767757.491915:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfed80. 00000100:00000001:3.0:1775767757.491917:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767757.491918:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767757.491920:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767757.491924:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767757.491926:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767757.491928:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767757.491929:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767758.513220:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767758.513226:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767758.513228:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767758.513244:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767758.513246:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767758.513247:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767758.513253:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767758.513254:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767758.513258:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767758.513258:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767758.513261:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767758.513264:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775767758.513265:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767758.513272:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767758.513275:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767758.513277:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767758.513279:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767758.513282:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767758.513293:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767758.513300:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767758.513303:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767758.513309:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767758.513314:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767758.513317:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767758.513320:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767758.513322:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767758.513325:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767758.577400:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767758.577404:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767758.577407:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775767758.833280:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299683646, at 4299683648 00000100:00000010:1.0:1775767758.833296:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942079816c0. 00000100:00000001:2.0:1775767759.537296:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767759.537305:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767759.537308:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767759.537314:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767759.537315:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767759.537315:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767759.537319:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767759.537329:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767759.537329:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767759.537329:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767759.537337:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767759.537338:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767759.537338:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767759.537346:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767759.537347:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775767759.537347:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767759.537353:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767759.537354:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767759.537355:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767759.537361:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767759.537362:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767759.537367:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767759.537371:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767759.537376:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767759.537376:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767759.537381:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767759.537386:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767760.561305:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767760.561311:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767760.561313:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767760.561318:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767760.561323:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767760.561323:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767760.561323:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767760.561332:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767760.561332:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767760.561333:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767760.561338:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767760.561339:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767760.561341:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767760.561348:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767760.561349:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:1.0:1775767760.561350:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767760.561362:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767760.561363:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767760.561363:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767760.561372:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767760.561373:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767760.561377:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767760.561379:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767760.561381:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767760.561387:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767760.561391:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767760.561394:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767760.625454:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767760.625460:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767760.625464:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775767761.585269:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767761.585269:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767761.585269:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767761.585275:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767761.585276:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767761.585276:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767761.585277:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767761.585280:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767761.585280:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767761.585284:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:1.0:1775767761.585286:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775767761.585287:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767761.585288:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767761.585292:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767761.585293:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767761.585293:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767761.585295:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767761.585297:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767761.585298:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767761.585300:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767761.585304:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767761.585304:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767761.585307:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767761.585309:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767761.585311:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767761.585314:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767761.585317:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767762.609312:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767762.609319:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767762.609322:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767762.609327:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767762.609333:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767762.609336:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767762.609337:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767762.609342:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767762.609343:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767762.609344:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767762.609347:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767762.609348:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767762.609354:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775767762.609358:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767762.609364:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767762.609369:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767762.609372:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767762.609376:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767762.609395:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767762.609401:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775767762.609409:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775767762.609413:0:36:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775767762.609420:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775767762.609430:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5015 00000400:00000200:1.0:1775767762.609441:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767762.609454:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775767762.609457:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775767762.609464:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767762.609468:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767762.609473:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767762.609476:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775767762.609479:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775767762.609482:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:1.0:1775767762.609484:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775767762.609487:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:1.0:1775767762.609489:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:1.0:1775767762.609491:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:1.0:1775767762.609494:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767762.609498:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767762.609503:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767762.609506:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775767762.609510:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:1.0:1775767762.609514:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775767762.609519:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767762.609522:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775767762.609527:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767762.609532:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767762.609535:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775767762.609544:0:36:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775767762.609551:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207889880. 00000020:00000040:1.0:1775767762.609554:0:36:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:1.0:1775767762.609557:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775767762.609559:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767762.609562:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775767762.609564:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775767762.609567:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775767762.609575:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543c800. 00000100:00000001:1.0:1775767762.609583:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767762.609593:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207889880 x1862022294231680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775767762.609602:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775767762.609607:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767762.609615:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f11856b8] to pc [ptlrpcd_rcv+-1] req@ffff994207889880 x1862022294231680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767762.609631:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767762.609636:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767762.609640:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775767762.609642:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775767762.609643:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775767762.609646:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775767762.609650:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5015 00000100:00000001:2.0:1775767762.609654:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:1.0:1775767762.609656:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767762.609660:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767762.609664:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000400:00000200:1.0:1775767762.609666:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775767762.609669:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775767762.609675:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207889880 x1862022294231680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775767762.609676:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767762.609681:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767762.609686:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767762.609692:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775767762.609698:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775767762.609701:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:1.0:1775767762.609704:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775767762.609707:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000100:00000001:2.0:1775767762.609708:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000001:1.0:1775767762.609710:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000001:2.0:1775767762.609711:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775767762.609713:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 02000000:00000001:2.0:1775767762.609718:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:1.0:1775767762.609720:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767762.609723:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1775767762.609724:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1775767762.609728:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000001:2.0:1775767762.609730:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767762.609731:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775767762.609735:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00100000:2.0:1775767762.609739:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207889880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294231680:192.168.202.156@tcp:250:kworker.0 00000100:00080000:1.0:1775767762.609740:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775767762.609745:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767762.609749:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775767762.609750:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775767762.609756:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767762.609756:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775767762.609759:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767762.609762:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:2.0:1775767762.609762:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767762.609767:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:1.0:1775767762.609768:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775767762.609773:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775767762.609774:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f400. 00000100:00000001:1.0:1775767762.609776:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767762.609779:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775767762.609782:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207888e00. 00000020:00000040:1.0:1775767762.609785:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:1.0:1775767762.609788:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775767762.609790:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767762.609790:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767762.609792:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775767762.609796:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775767762.609800:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775767762.609800:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b391b8. 02000000:00000010:1.0:1775767762.609808:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543c400. 00000400:00000010:2.0:1775767762.609810:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f770. 00000100:00000001:1.0:1775767762.609811:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767762.609818:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207888e00 x1862022294231808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:2.0:1775767762.609821:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294231680, portal 25 00000100:00000001:1.0:1775767762.609826:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775767762.609827:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775767762.609828:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767762.609830:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606124672 : -112966103426944 : ffff994207889880) 00000100:00000040:1.0:1775767762.609832:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001812f966] to pc [ptlrpcd_rcv+-1] req@ffff994207888e00 x1862022294231808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767762.609839:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767762.609841:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767762.609843:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775767762.609846:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775767762.609848:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775767762.609851:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00000040:2.0:1775767762.609853:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207889880 x1862022294231680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767787 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767762.609854:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5015 00000400:00000200:1.0:1775767762.609862:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767762.609868:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775767762.609872:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1775767762.609874:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775767762.609876:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767762.609879:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767762.609882:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000040:2.0:1775767762.609882:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000100:00000001:1.0:1775767762.609885:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775767762.609887:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000400:00000010:2.0:1775767762.609887:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f990. 00000020:00000040:1.0:1775767762.609888:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:1.0:1775767762.609891:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775767762.609893:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:1.0:1775767762.609895:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000200:2.0:1775767762.609895:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294231680, offset 0 00000020:00000001:1.0:1775767762.609898:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:1.0:1775767762.609902:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767762.609903:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767762.609907:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767762.609909:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000200:2.0:1775767762.609909:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000020:00000040:1.0:1775767762.609912:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:1.0:1775767762.609915:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775767762.609919:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767762.609920:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775767762.609923:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1775767762.609923:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:1.0:1775767762.609927:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767762.609929:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775767762.609930:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775767762.609932:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767762.609934:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767762.609936:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00000010:1.0:1775767762.609938:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff99420788b100. 00000020:00000040:1.0:1775767762.609941:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000400:00000200:2.0:1775767762.609943:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:1.0:1775767762.609944:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775767762.609946:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767762.609948:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775767762.609949:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775767762.609952:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767762.609952:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 02000000:00000010:1.0:1775767762.609957:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543e800. 00000400:00000200:2.0:1775767762.609960:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000100:00000001:1.0:1775767762.609961:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767762.609969:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00100000:1.0:1775767762.609971:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff99420788b100 x1862022294231936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775767762.609980:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:2.0:1775767762.609981:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224983:224983:256:4294967295] 192.168.202.156@tcp LPNI seq info [224979:224979:8:4294967295] 00000100:00000001:1.0:1775767762.609984:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767762.609991:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e26cf977] to pc [ptlrpcd_rcv+-1] req@ffff99420788b100 x1862022294231936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1775767762.609992:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767762.610002:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767762.610005:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767762.610010:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775767762.610014:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775767762.610016:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775767762.610019:0:36:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:2.0:1775767762.610019:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000100:00080000:1.0:1775767762.610024:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5015 00000400:00000200:1.0:1775767762.610028:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000800:00000200:2.0:1775767762.610032:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775767762.610033:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775767762.610037:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775767762.610039:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767762.610042:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767762.610043:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767762.610045:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775767762.610046:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775767762.610048:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:1.0:1775767762.610050:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000800:00000010:2.0:1775767762.610050:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3a00. 00000020:00000040:1.0:1775767762.610053:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:1.0:1775767762.610056:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:1.0:1775767762.610058:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000800:00000200:2.0:1775767762.610065:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775767762.610072:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000100:00000040:1.0:1775767762.610073:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767762.610078:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767762.610080:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767762.610082:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775767762.610085:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000800:00000200:2.1:1775767762.610087:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00080000:1.0:1775767762.610090:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775767762.610095:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775767762.610096:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 02000000:00000001:1.0:1775767762.610099:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775767762.610102:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767762.610105:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1775767762.610107:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000100:00000001:1.0:1775767762.610108:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775767762.610111:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000800:00000010:2.0:1775767762.610112:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3a00. 00000008:00000001:1.0:1775767762.610113:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767762.610116:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775767762.610119:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207889f80. 00000020:00000040:1.0:1775767762.610121:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00000400:00000200:2.0:1775767762.610122:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000001:1.0:1775767762.610123:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775767762.610124:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767762.610127:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775767762.610128:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775767762.610132:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767762.610132:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 02000000:00000010:1.0:1775767762.610139:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543fc00. 00000400:00000200:2.0:1775767762.610141:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f990 00000100:00000001:1.0:1775767762.610143:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775767762.610145:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f990. 00000100:00100000:1.0:1775767762.610151:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207889f80 x1862022294232064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775767762.610151:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767762.610186:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207889880 x1862022294231680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767787 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767762.610218:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775767762.610222:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767762.610226:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767762.610229:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ed2274f4] to pc [ptlrpcd_rcv+-1] req@ffff994207889f80 x1862022294232064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775767762.610237:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207889880 x1862022294231680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767787 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767762.610239:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767762.610246:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767762.610254:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1775767762.610255:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767762.610256:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775767762.610262:0:36:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775767762.610262:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775767762.610266:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775767762.610271:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5015 00000100:00000001:2.0:1775767762.610277:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767762.610280:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767762.610289:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000400:00000200:2.0:1775767762.610290:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00080000:1.0:1775767762.610293:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1775767762.610296:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299688766 00000100:00000040:1.0:1775767762.610297:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767762.610302:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767762.610303:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000010:2.0:1775767762.610304:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d6000. 00000100:00000001:1.0:1775767762.610305:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775767762.610310:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775767762.610311:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:1.0:1775767762.610312:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775767762.610313:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:1.0:1775767762.610315:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:1.0:1775767762.610316:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000001:2.0:1775767762.610316:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000040:1.0:1775767762.610321:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767762.610323:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767762.610325:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767762.610326:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299688766 00000100:00000001:1.0:1775767762.610327:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775767762.610329:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:1.0:1775767762.610332:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000010:2.0:1775767762.610332:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d6c60. 00000100:00000001:1.0:1775767762.610333:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767762.610338:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1775767762.610340:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 02000000:00000001:1.0:1775767762.610341:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767762.610344:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767762.610346:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775767762.610347:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000100:00000001:2.0:1775767762.610347:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00000001:1.0:1775767762.610352:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767762.610354:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767762.610355:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775767762.610361:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207888700. 00000100:00000001:2.0:1775767762.610361:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000020:00000040:1.0:1775767762.610365:0:36:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:1.0:1775767762.610367:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775767762.610369:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767762.610370:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767762/real 1775767762] req@ffff994207889880 x1862022294231680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767762.610371:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775767762.610373:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775767762.610375:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775767762.610379:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543f800. 00000100:00000001:1.0:1775767762.610384:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767762.610391:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207888700 x1862022294232192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:2.0:1775767762.610392:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767762.610395:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1775767762.610399:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000010:2.0:1775767762.610399:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b391b8. 00000100:00000001:1.0:1775767762.610402:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767762.610405:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f770 00000100:00000040:1.0:1775767762.610406:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009f2ba301] to pc [ptlrpcd_rcv+-1] req@ffff994207888700 x1862022294232192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1775767762.610410:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f770. 00000100:00000001:1.0:1775767762.610416:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767762.610418:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:1.0:1775767762.610420:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5025) 00000100:00000200:2.0:1775767762.610426:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207889880 x1862022294231680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767762.610438:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207889880 x1862022294231680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767762.610446:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767762.610448:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767762.610451:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767762.610453:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767762.610458:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207889880 x1862022294231680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767762.610467:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767762.610471:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207889880 x1862022294231680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767787 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767762.610479:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767762.610482:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767762.610484:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767762.610485:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767762.610488:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767762.610490:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767762.610493:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767762.610497:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767762.610501:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767762.610506:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207889880 x1862022294231680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767787 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767762.610515:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207889880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294231680:192.168.202.156@tcp:250:kworker.0 00000100:00000001:2.0:1775767762.610520:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767762.610522:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767762.610526:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207889880 x1862022294231680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767787 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767762.610535:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767762.610537:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767762.610540:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f400. 02000000:00000001:2.0:1775767762.610543:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767762.610545:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767762.610547:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:2.0:1775767762.610549:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767762.610552:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543c800. 02000000:00000001:2.0:1775767762.610555:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767762.610558:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767762.610560:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207889880. 00000100:00000001:2.0:1775767762.610563:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767762.610565:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767762.610567:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767762.610574:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767762.610576:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767762.610578:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775767762.610580:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767762.610581:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767762.610585:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207888e00 x1862022294231808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767762.610593:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767762.610594:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767762.610596:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767762.610598:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767762.610601:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767762.610604:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207888e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294231808:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775767762.610608:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767762.610610:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767762.610611:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767762.610613:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775767762.610617:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f400. 02000000:00000001:2.0:1775767762.610619:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767762.610623:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767762.610626:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b391b8. 00000400:00000010:2.0:1775767762.610630:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f770. 00000100:00000200:2.0:1775767762.610634:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294231808, portal 10 00000100:00000001:2.0:1775767762.610636:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767762.610637:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606121984 : -112966103429632 : ffff994207888e00) 00000100:00000040:2.0:1775767762.610645:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207888e00 x1862022294231808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767787 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767762.610652:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767762.610655:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767762.610658:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f990. 00000100:00000200:2.0:1775767762.610661:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294231808, offset 0 00000400:00000200:2.0:1775767762.610666:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767762.610671:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767762.610675:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767762.610678:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767762.610681:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767762.610683:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767762.610686:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767762.610689:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224984:224984:256:4294967295] 192.168.202.156@tcp LPNI seq info [224980:224980:8:4294967295] 00000400:00000200:2.0:1775767762.610694:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767762.610701:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767762.610706:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767762.610710:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3a00. 00000800:00000200:2.0:1775767762.610715:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775767762.610718:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775767762.610724:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767762.610727:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000100:2.0:1775767762.610732:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767762.610734:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3a00. 00000400:00000200:2.0:1775767762.610737:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775767762.610741:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775767762.610745:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f990 00000400:00000010:2.0:1775767762.610747:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f990. 00000100:00000001:2.0:1775767762.610749:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767762.610754:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207888e00 x1862022294231808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767787 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767762.610763:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767762.610767:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207888e00 x1862022294231808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767787 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767762.610774:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767762.610775:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775767762.610779:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767762.610783:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:2.0:1775767762.610786:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767762.610788:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299687443 00000100:00000001:2.0:1775767762.610792:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299688766 00000100:00000001:2.0:1775767762.610794:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299687443 00000100:00000001:2.0:1775767762.610796:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299688766 00000100:00000001:2.0:1775767762.610798:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767762.610800:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767762.610803:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767762.610807:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767762/real 1775767762] req@ffff994207888e00 x1862022294231808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775767762.610823:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767762.610825:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767762.610826:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b391b8. 00000400:00000200:2.0:1775767762.610830:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f770 00000400:00000010:2.0:1775767762.610831:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f770. 00000100:00000001:2.0:1775767762.610834:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767762.610838:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207888e00 x1862022294231808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767762.610847:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207888e00 x1862022294231808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767762.610854:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767762.610856:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767762.610858:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767762.610859:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767762.610863:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207888e00 x1862022294231808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767762.610870:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767762.610874:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207888e00 x1862022294231808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767787 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767762.610881:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767762.610884:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767762.610886:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767762.610888:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767762.610890:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767762.610892:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767762.610896:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767762.610899:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767762.610902:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767762.610908:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207888e00 x1862022294231808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767787 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767762.610917:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207888e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294231808:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775767762.610921:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767762.610925:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff99420788b100 x1862022294231936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767762.610933:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767762.610935:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767762.610937:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767762.610940:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767762.610942:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767762.610945:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff99420788b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294231936:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775767762.610948:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767762.610950:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767762.610952:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767762.610953:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775767762.610956:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f000. 02000000:00000001:2.0:1775767762.610958:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767762.610962:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767762.610965:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b391b8. 00000400:00000010:2.0:1775767762.610968:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f770. 00000100:00000200:2.0:1775767762.610971:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294231936, portal 10 00000100:00000001:2.0:1775767762.610972:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767762.610974:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606130944 : -112966103420672 : ffff99420788b100) 00000100:00000040:2.0:1775767762.610980:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff99420788b100 x1862022294231936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767787 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767762.610987:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767762.610990:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767762.610992:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f990. 00000100:00000200:2.0:1775767762.610995:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294231936, offset 0 00000400:00000200:2.0:1775767762.610999:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767762.611004:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767762.611008:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767762.611011:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767762.611014:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767762.611016:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767762.611019:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767762.611022:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224985:224985:256:4294967295] 192.168.202.156@tcp LPNI seq info [224981:224981:8:4294967295] 00000400:00000200:2.0:1775767762.611027:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767762.611033:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767762.611038:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767762.611042:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3a00. 00000800:00000200:2.0:1775767762.611046:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775767762.611049:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775767762.611053:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767762.611056:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000100:2.0:1775767762.611060:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767762.611062:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3a00. 00000400:00000200:2.0:1775767762.611065:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775767762.611069:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775767762.611072:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f990 00000400:00000010:2.0:1775767762.611074:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f990. 00000100:00000001:2.0:1775767762.611076:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767762.611080:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff99420788b100 x1862022294231936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767787 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767762.611089:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767762.611093:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff99420788b100 x1862022294231936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767787 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767762.611101:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767762.611103:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775767762.611106:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767762.611109:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:2.0:1775767762.611112:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767762.611116:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299687435 00000100:00000001:2.0:1775767762.611118:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299688766 00000100:00000001:2.0:1775767762.611121:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299687435 00000100:00000001:2.0:1775767762.611123:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299688766 00000100:00000001:2.0:1775767762.611125:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767762.611127:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767762.611129:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767762.611133:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767762/real 1775767762] req@ffff99420788b100 x1862022294231936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775767762.611141:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767762.611143:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767762.611144:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b391b8. 00000400:00000200:2.0:1775767762.611147:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f770 00000400:00000010:2.0:1775767762.611148:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f770. 00000100:00000001:2.0:1775767762.611151:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767762.611185:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff99420788b100 x1862022294231936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767762.611211:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff99420788b100 x1862022294231936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767762.611218:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767762.611220:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767762.611222:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767762.611224:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767762.611228:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff99420788b100 x1862022294231936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767762.611235:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767762.611239:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff99420788b100 x1862022294231936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767787 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767762.611247:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767762.611250:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767762.611252:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767762.611254:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767762.611255:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767762.611259:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767762.611262:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767762.611265:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767762.611267:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767762.611271:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff99420788b100 x1862022294231936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767787 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767762.611280:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff99420788b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294231936:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775767762.611284:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767762.611287:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207889f80 x1862022294232064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767762.611294:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767762.611296:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767762.611298:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767762.611300:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767762.611302:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767762.611305:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207889f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294232064:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767762.611309:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767762.611311:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767762.611313:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767762.611315:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775767762.611318:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1e800. 02000000:00000001:2.0:1775767762.611320:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767762.611324:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767762.611327:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b391b8. 00000400:00000010:2.0:1775767762.611330:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f770. 00000100:00000200:2.0:1775767762.611333:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294232064, portal 4 00000100:00000001:2.0:1775767762.611336:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767762.611337:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606126464 : -112966103425152 : ffff994207889f80) 00000100:00000040:2.0:1775767762.611343:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207889f80 x1862022294232064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767787 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767762.611350:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767762.611353:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767762.611355:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f990. 00000100:00000200:2.0:1775767762.611358:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294232064, offset 0 00000400:00000200:2.0:1775767762.611362:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767762.611366:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767762.611370:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767762.611373:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767762.611376:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767762.611379:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767762.611381:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767762.611384:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224986:224986:256:4294967295] 192.168.202.156@tcp LPNI seq info [224982:224982:8:4294967295] 00000400:00000200:2.0:1775767762.611389:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767762.611395:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767762.611400:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767762.611403:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3a00. 00000800:00000200:2.0:1775767762.611407:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775767762.611410:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775767762.611414:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767762.611417:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000100:2.0:1775767762.611421:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767762.611422:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3a00. 00000400:00000200:2.0:1775767762.611425:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775767762.611429:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775767762.611432:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f990 00000400:00000010:2.0:1775767762.611434:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f990. 00000100:00000001:2.0:1775767762.611436:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767762.611439:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207889f80 x1862022294232064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767787 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767762.611454:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767762.611457:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207889f80 x1862022294232064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767787 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767762.611464:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767762.611465:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775767762.611469:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767762.611472:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:2.0:1775767762.611475:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767762.611477:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299687459 00000100:00000001:2.0:1775767762.611480:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299688766 00000100:00000001:2.0:1775767762.611482:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299687459 00000100:00000001:2.0:1775767762.611484:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299688766 00000100:00000001:2.0:1775767762.611486:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767762.611488:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767762.611490:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767762.611494:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767762/real 1775767762] req@ffff994207889f80 x1862022294232064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775767762.611503:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767762.611504:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767762.611506:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b391b8. 00000400:00000200:2.0:1775767762.611509:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f770 00000400:00000010:2.0:1775767762.611510:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f770. 00000100:00000001:2.0:1775767762.611513:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767762.611516:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207889f80 x1862022294232064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767762.611525:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207889f80 x1862022294232064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767762.611532:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767762.611534:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767762.611536:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767762.611537:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767762.611541:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207889f80 x1862022294232064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767762.611548:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767762.611553:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207889f80 x1862022294232064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767787 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767762.611560:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767762.611563:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767762.611564:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767762.611566:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767762.611567:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767762.611571:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767762.611573:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767762.611577:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767762.611580:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767762.611584:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207889f80 x1862022294232064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767787 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767762.611593:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207889f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294232064:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767762.611597:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767762.611601:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207888700 x1862022294232192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767762.611607:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767762.611609:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767762.611611:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767762.611613:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767762.611615:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767762.611619:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207888700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294232192:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767762.611622:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767762.611624:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767762.611626:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767762.611627:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775767762.611630:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1ec00. 02000000:00000001:2.0:1775767762.611632:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767762.611636:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767762.611638:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b391b8. 00000400:00000010:2.0:1775767762.611641:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f770. 00000100:00000200:2.0:1775767762.611644:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294232192, portal 4 00000100:00000001:2.0:1775767762.611646:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767762.611648:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606120192 : -112966103431424 : ffff994207888700) 00000100:00000040:2.0:1775767762.611653:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207888700 x1862022294232192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767787 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767762.611660:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767762.611663:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767762.611665:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f990. 00000100:00000200:2.0:1775767762.611667:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294232192, offset 0 00000400:00000200:2.0:1775767762.611671:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767762.611675:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767762.611679:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767762.611682:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767762.611684:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767762.611694:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767762.611696:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767762.611700:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224987:224987:256:4294967295] 192.168.202.156@tcp LPNI seq info [224983:224983:8:4294967295] 00000400:00000200:2.0:1775767762.611704:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767762.611711:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767762.611716:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767762.611720:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3a00. 00000800:00000200:2.0:1775767762.611724:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775767762.611727:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775767762.611732:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767762.611735:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000100:2.0:1775767762.611739:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767762.611741:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3a00. 00000400:00000200:2.0:1775767762.611744:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775767762.611748:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775767762.611751:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f990 00000400:00000010:2.0:1775767762.611753:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f990. 00000100:00000001:2.0:1775767762.611755:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767762.611759:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207888700 x1862022294232192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767787 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767762.611767:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767762.611771:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207888700 x1862022294232192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767787 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767762.611778:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767762.611780:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775767762.611784:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767762.611787:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:2.0:1775767762.611791:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767762.611793:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299687462 00000100:00000001:2.0:1775767762.611796:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299688766 00000100:00000001:2.0:1775767762.611798:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299687462 00000100:00000001:2.0:1775767762.611800:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299688766 00000100:00000001:2.0:1775767762.611803:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767762.611805:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767762.611807:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767762.611811:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767762/real 1775767762] req@ffff994207888700 x1862022294232192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775767762.611819:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767762.611821:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767762.611823:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b391b8. 00000400:00000200:2.0:1775767762.611825:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f770 00000400:00000010:2.0:1775767762.611827:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f770. 00000100:00000001:2.0:1775767762.611830:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767762.611834:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207888700 x1862022294232192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767762.611843:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207888700 x1862022294232192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767762.611850:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767762.611852:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767762.611853:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767762.611855:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767762.611859:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207888700 x1862022294232192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767762.611866:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767762.611870:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207888700 x1862022294232192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767787 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767762.611878:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767762.611880:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767762.611882:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767762.611883:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767762.611885:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767762.611889:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767762.611892:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767762.611895:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767762.611898:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767762.611902:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207888700 x1862022294232192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767787 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767762.611917:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207888700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294232192:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767762.611921:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767762.611923:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767762.611926:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207888e00 x1862022294231808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767787 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767762.611934:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767762.611936:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767762.611938:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f400. 02000000:00000001:2.0:1775767762.611941:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767762.611943:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767762.611944:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:2.0:1775767762.611946:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767762.611948:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543c400. 02000000:00000001:2.0:1775767762.611950:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767762.611952:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767762.611953:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207888e00. 00000100:00000001:2.0:1775767762.611956:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767762.611957:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767762.611959:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767762.611962:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff99420788b100 x1862022294231936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767787 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767762.611969:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767762.611970:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767762.611972:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f000. 02000000:00000001:2.0:1775767762.611975:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767762.611976:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767762.611977:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:2.0:1775767762.611979:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767762.611981:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543e800. 02000000:00000001:2.0:1775767762.611983:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767762.611984:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767762.611986:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff99420788b100. 00000100:00000001:2.0:1775767762.611989:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767762.611990:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767762.611992:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767762.612019:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207889f80 x1862022294232064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767787 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767762.612027:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767762.612028:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767762.612029:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1e800. 02000000:00000001:2.0:1775767762.612033:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767762.612034:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767762.612035:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:2.0:1775767762.612037:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767762.612039:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543fc00. 02000000:00000001:2.0:1775767762.612042:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767762.612043:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767762.612044:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207889f80. 00000100:00000001:2.0:1775767762.612047:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767762.612048:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767762.612051:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767762.612054:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207888700 x1862022294232192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767787 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767762.612061:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767762.612063:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767762.612065:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1ec00. 02000000:00000001:2.0:1775767762.612069:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767762.612072:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767762.612073:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:2.0:1775767762.612075:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767762.612077:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543f800. 02000000:00000001:2.0:1775767762.612081:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767762.612082:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767762.612085:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207888700. 00000100:00000001:2.0:1775767762.612087:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767762.612089:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767762.612091:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767762.612095:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767762.612097:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767762.612099:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775767762.612100:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767762.673404:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767762.673409:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767762.673413:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775767763.633295:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767763.633302:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767763.633307:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767763.633309:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767763.633312:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767763.633317:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767763.633320:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:1.0:1775767763.633321:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767763.633326:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767763.633329:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767763.633329:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767763.633334:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767763.633338:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767763.633338:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767763.633347:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767763.633349:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767763.633353:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767763.633358:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767763.953247:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299688766, at 4299688768 00000100:00000010:1.0:1775767763.953258:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d6c60. 00000100:00000001:1.0:1775767764.657288:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767764.657289:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767764.657295:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767764.657296:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767764.657298:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767764.657300:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767764.657306:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775767764.657307:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767764.657313:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767764.657313:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767764.657319:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767764.657319:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767764.657332:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767764.657334:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767764.657337:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767764.657339:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767764.657343:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767764.657346:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767764.721326:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767764.721329:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767764.721332:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775767765.681285:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767765.681292:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767765.681293:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767765.681294:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767765.681296:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:1.0:1775767765.681299:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775767765.681300:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767765.681301:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767765.681305:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767765.681305:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767765.681306:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767765.681309:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767765.681313:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00100000:2.0:1775767765.681313:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775767765.681313:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767765.681314:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767765.681319:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767765.681320:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767765.681321:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767765.681323:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767765.681325:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767765.681328:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767765.681330:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767765.681332:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767765.681338:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767765.681341:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767765.681344:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767766.705225:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767766.705225:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767766.705232:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767766.705232:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767766.705234:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767766.705234:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767766.705240:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775767766.705240:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767766.705243:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767766.705245:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767766.705245:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767766.705247:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767766.705250:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767766.705250:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767766.705250:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767766.705253:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767766.705255:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767766.705257:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767766.705259:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767766.705260:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767766.705265:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767766.705265:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767766.705267:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767766.705269:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767766.705270:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767766.705273:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767766.705276:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767766.769319:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767766.769323:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767766.769327:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775767767.729268:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767767.729275:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767767.729278:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767767.729281:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767767.729284:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:3.0:1775767767.729284:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767767.729286:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767767.729289:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767767.729289:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767767.729293:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767767.729294:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767767.729296:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767767.729297:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767767.729301:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767767.729306:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767767.729306:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767767.729308:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767767.729313:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767767.729418:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767767.729422:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767767.729425:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767767.729429:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767767.729434:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767767.729437:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767767.729441:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767767.729444:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767767.729447:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767767.729544:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775767767.729547:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767767.729549:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767767.729558:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fd500 x1862022294232320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767767.729569:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767767.729572:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767767.729576:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767767.729579:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767767.729584:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767767.729588:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294232320:192.168.202.156@tcp:250:kworker.0 00000100:00000001:2.0:1775767767.729593:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767767.729598:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767767.729601:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767767.729606:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775767767.729611:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1ec00. 02000000:00000001:2.0:1775767767.729616:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767767.729626:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767767.729634:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b391b8. 00000400:00000010:2.0:1775767767.729645:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f770. 00000100:00000200:2.0:1775767767.729653:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294232320, portal 25 00000100:00000001:2.0:1775767767.729659:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767767.729663:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641201920 : -112966068349696 : ffff9942099fd500) 00000100:00000040:2.0:1775767767.729673:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fd500 x1862022294232320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767792 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767767.729691:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767767.729696:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767767.729700:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f990. 00000100:00000200:2.0:1775767767.729703:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294232320, offset 0 00000400:00000200:2.0:1775767767.729709:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767767.729718:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767767.729724:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767767.729728:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767767.729735:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767767.729740:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767767.729745:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767767.729751:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224988:224988:256:4294967295] 192.168.202.156@tcp LPNI seq info [224984:224984:8:4294967295] 00000400:00000200:2.0:1775767767.729760:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767767.729775:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767767.729792:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767767.729798:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3a00. 00000800:00000200:2.0:1775767767.729807:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767767.729814:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:2.0:1775767767.729826:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767767.729836:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767767.729841:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299693885 00000100:00000010:2.0:1775767767.729847:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d64e0. 00000100:00000001:2.0:1775767767.729855:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1775767767.729862:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299693885 00000100:00000010:2.0:1775767767.729866:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d6240. 00000100:00000001:2.0:1775767767.729870:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1775767767.729875:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767767.729880:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767767.729885:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767767.729890:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767767.729897:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767767.729900:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1775767767.729903:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775767767.729906:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767767.729908:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767767.729913:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ff800 x1862022294232448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767767.729927:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767767.729929:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767767.729933:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767767.729935:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767767.729937:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767767.729943:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294232448:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775767767.729950:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767767.729952:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767767.729954:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767767.729958:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775767767.729963:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1e800. 02000000:00000001:2.0:1775767767.729965:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767767.729970:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767767.729973:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39318. 00000400:00000010:2.0:1775767767.729978:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f330. 00000100:00000200:2.0:1775767767.729983:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294232448, portal 10 00000100:00000001:2.0:1775767767.729987:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767767.729989:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641210880 : -112966068340736 : ffff9942099ff800) 00000100:00000040:2.0:1775767767.729997:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ff800 x1862022294232448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767792 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767767.730006:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767767.730009:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767767.730012:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8faa0. 00000100:00000200:2.0:1775767767.730015:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294232448, offset 0 00000400:00000200:2.0:1775767767.730020:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767767.730025:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767767.730031:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767767.730036:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767767.730041:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767767.730046:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767767.730048:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767767.730053:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224989:224989:255:4294967295] 192.168.202.156@tcp LPNI seq info [224985:224985:7:4294967295] 00000400:00000200:2.0:1775767767.730071:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767767.730083:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767767.730088:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767767.730092:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3500. 00000800:00000200:2.0:1775767767.730100:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767767.730109:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:2.0:1775767767.730117:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767767.730123:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:2.0:1775767767.730131:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767767.730135:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299692562 00000100:00000001:2.0:1775767767.730198:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299693885 00000100:00000001:2.0:1775767767.730204:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299692563 00000100:00000001:2.0:1775767767.730208:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299693885 00000100:00000001:2.0:1775767767.730213:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767767.730218:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767767.730224:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767767.730229:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767767.730236:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767767.730240:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1775767767.730245:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775767767.730248:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767767.730251:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767767.730260:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fc000 x1862022294232576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767767.730278:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767767.730281:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767767.730287:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767767.730292:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767767.730297:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767767.730305:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294232576:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775767767.730314:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767767.730317:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767767.730320:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767767.730324:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775767767.730331:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f000. 02000000:00000001:2.0:1775767767.730336:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767767.730343:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767767.730349:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39bb0. 00000400:00000010:2.0:1775767767.730355:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8fcc0. 00000100:00000200:2.0:1775767767.730361:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294232576, portal 10 00000100:00000001:2.0:1775767767.730367:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767767.730370:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641196544 : -112966068355072 : ffff9942099fc000) 00000100:00000040:2.0:1775767767.730381:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fc000 x1862022294232576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767792 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767767.730401:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767767.730407:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767767.730410:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8fdd0. 00000100:00000200:2.0:1775767767.730414:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294232576, offset 0 00000400:00000200:2.0:1775767767.730425:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767767.730434:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767767.730440:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767767.730448:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767767.730455:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767767.730461:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767767.730466:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767767.730475:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224990:224990:254:4294967295] 192.168.202.156@tcp LPNI seq info [224986:224986:6:4294967295] 00000400:00000200:2.0:1775767767.730487:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767767.730501:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767767.730513:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767767.730520:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3800. 00000800:00000200:2.0:1775767767.730529:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767767.730549:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:2.0:1775767767.730554:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767767.730560:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:2.0:1775767767.730569:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767767.730573:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299692555 00000100:00000001:2.0:1775767767.730579:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299693885 00000100:00000001:2.0:1775767767.730585:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299692555 00000100:00000001:2.0:1775767767.730591:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299693885 00000100:00000001:2.0:1775767767.730594:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767767.730599:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767767.730605:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767767.730609:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767767.730617:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767767.730621:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1775767767.730625:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775767767.730628:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767767.730630:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767767.730636:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ff100 x1862022294232704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767767.730650:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767767.730652:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767767.730658:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767767.730661:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767767.730665:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767767.730672:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294232704:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767767.730679:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767767.730682:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767767.730685:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767767.730690:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775767767.730696:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f400. 02000000:00000001:2.0:1775767767.730701:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767767.730709:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767767.730715:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39630. 00000400:00000010:2.0:1775767767.730719:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f440. 00000100:00000200:2.0:1775767767.730722:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294232704, portal 4 00000100:00000001:2.0:1775767767.730724:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767767.730725:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641209088 : -112966068342528 : ffff9942099ff100) 00000100:00000040:2.0:1775767767.730732:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ff100 x1862022294232704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767792 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767767.730739:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767767.730742:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767767.730745:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775767767.730748:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294232704, offset 0 00000400:00000200:2.0:1775767767.730752:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767767.730757:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767767.730760:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767767.730764:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767767.730766:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767767.730769:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767767.730772:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767767.730775:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224991:224991:253:4294967295] 192.168.202.156@tcp LPNI seq info [224987:224987:5:4294967295] 00000400:00000200:2.0:1775767767.730779:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767767.730785:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767767.730790:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767767.730794:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3600. 00000800:00000200:2.0:1775767767.730798:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767767.730801:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:2.0:1775767767.730803:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767767.730806:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:2.0:1775767767.730809:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767767.730811:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299692578 00000100:00000001:2.0:1775767767.730815:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299693885 00000100:00000001:2.0:1775767767.730817:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299692578 00000100:00000001:2.0:1775767767.730819:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299693885 00000100:00000001:2.0:1775767767.730821:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767767.730823:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767767.730826:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767767.730828:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767767.730831:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767767.730832:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1775767767.730835:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775767767.730836:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767767.730837:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767767.730841:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fc700 x1862022294232832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767767.730848:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767767.730850:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767767.730852:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767767.730854:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767767.730856:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767767.730858:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294232832:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767767.730861:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767767.730862:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767767.730864:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767767.730866:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775767767.730869:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1c800. 02000000:00000001:2.0:1775767767.730871:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767767.730874:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767767.730878:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39e70. 00000400:00000010:2.0:1775767767.730881:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8ff68. 00000100:00000200:2.0:1775767767.730884:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294232832, portal 4 00000100:00000001:2.0:1775767767.730886:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767767.730887:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641198336 : -112966068353280 : ffff9942099fc700) 00000100:00000040:2.0:1775767767.730892:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fc700 x1862022294232832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767792 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767767.730899:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767767.730902:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767767.730905:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f220. 00000100:00000200:2.0:1775767767.730907:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294232832, offset 0 00000400:00000200:2.0:1775767767.730918:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767767.730922:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767767.730925:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767767.730928:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767767.730930:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767767.730933:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767767.730935:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767767.730938:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224992:224992:252:4294967295] 192.168.202.156@tcp LPNI seq info [224988:224988:4:4294967295] 00000400:00000200:2.0:1775767767.730942:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767767.730947:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767767.730952:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767767.730955:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3300. 00000800:00000200:2.0:1775767767.730958:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767767.730962:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:2.0:1775767767.730964:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767767.730966:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:2.0:1775767767.730969:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767767.730971:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299692581 00000100:00000001:2.0:1775767767.730973:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299693885 00000100:00000001:2.0:1775767767.730975:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299692581 00000100:00000001:2.0:1775767767.730977:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299693885 00000100:00000001:2.0:1775767767.730978:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767767.730980:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767767.730983:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767767.730984:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767767.730987:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767767.730989:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1775767767.730991:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775767767.730992:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767767.730994:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767767.730995:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767768.753233:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767768.753240:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767768.753242:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767768.753246:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767768.753256:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767768.753260:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767768.753262:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767768.753265:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767768.753269:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767768.753304:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767768.753308:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767768.753310:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767768.753314:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767768.753316:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767768.753318:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767768.753320:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767768.753321:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767768.753323:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767768.817329:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767768.817332:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767768.817335:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775767769.073230:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299693885, at 4299693888 00000100:00000010:1.0:1775767769.073249:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d6240. 00000100:00000001:3.0:1775767769.777251:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767769.777256:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767769.777258:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767769.777264:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767769.777268:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767769.777271:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767769.777273:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767769.777276:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767769.777280:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767769.777312:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767769.777316:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767769.777319:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767769.777319:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767769.777328:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:1.0:1775767769.777329:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775767769.777332:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767769.777336:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:2.0:1775767769.777339:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767769.777340:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767769.777349:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767769.777350:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767769.777356:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767769.777358:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767769.777360:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767769.777361:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767769.777365:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767769.777367:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767770.801207:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767770.801215:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767770.801217:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767770.801222:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767770.801231:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767770.801234:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767770.801236:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767770.801237:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767770.801241:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767770.801270:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767770.801275:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767770.801276:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767770.801281:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767770.801285:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767770.801289:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767770.801291:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767770.801293:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767770.801295:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767770.801357:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767770.801360:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767770.801361:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767770.801364:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767770.801366:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767770.801368:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767770.801370:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767770.801371:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767770.801372:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:2.0:1775767770.801454:0:2387:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.156:988 00000400:00000100:2.0:1775767770.801506:0:2387:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.156@tcp at host 192.168.202.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:2.0:1775767770.801518:0:2387:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767770.801521:0:2387:0:(router.c:1750:lnet_notify()) 192.168.202.56@tcp notifying 192.168.202.156@tcp: down 00000400:00000200:2.0:1775767770.801526:0:2387:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767770.801530:0:2387:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:2.0:1775767770.801535:0:2387:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000800:00000010:2.0:1775767770.801539:0:2387:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3a00. 00000400:00000200:2.0:1775767770.801545:0:2387:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775767770.801550:0:2387:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775767770.801557:0:2387:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f990 00000400:00000010:2.0:1775767770.801568:0:2387:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f990. 00000100:00000001:2.0:1775767770.801574:0:2387:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767770.801586:0:2387:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9942099fd500 x1862022294232320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767792 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767770.801625:0:2387:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767770.801630:0:2387:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fd500 x1862022294232320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767792 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767770.801639:0:2387:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767770.801641:0:2387:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1775767770.801655:0:2387:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000800:00000010:2.0:1775767770.801657:0:2387:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3500. 00000400:00000200:2.0:1775767770.801660:0:2387:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775767770.801663:0:2387:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775767770.801666:0:2387:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8faa0 00000400:00000010:2.0:1775767770.801668:0:2387:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8faa0. 00000100:00000001:2.0:1775767770.801670:0:2387:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767770.801675:0:2387:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9942099ff800 x1862022294232448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767792 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767770.801683:0:2387:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767770.801685:0:2387:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ff800 x1862022294232448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767792 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767770.801691:0:2387:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767770.801693:0:2387:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1775767770.801698:0:2387:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000800:00000010:2.0:1775767770.801700:0:2387:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3800. 00000400:00000200:2.0:1775767770.801703:0:2387:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775767770.801706:0:2387:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775767770.801709:0:2387:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8fdd0 00000400:00000010:2.0:1775767770.801710:0:2387:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8fdd0. 00000100:00000001:2.0:1775767770.801713:0:2387:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767770.801718:0:2387:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9942099fc000 x1862022294232576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767792 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767770.801726:0:2387:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767770.801729:0:2387:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fc000 x1862022294232576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767792 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767770.801736:0:2387:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767770.801738:0:2387:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1775767770.801743:0:2387:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000800:00000010:2.0:1775767770.801745:0:2387:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3600. 00000100:00000001:3.0:1775767770.802319:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767770.802329:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767770.802334:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767770.802347:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767767/real 1775767770] req@ffff9942099fc000 x1862022294232576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767770.802363:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767770.802366:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767770.802369:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39bb0. 00000400:00000200:3.0:1775767770.802374:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8fcc0 00000400:00000010:3.0:1775767770.802376:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8fcc0. 00000100:00000001:3.0:1775767770.802379:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767770.802383:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fc000 x1862022294232576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767770.802393:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fc000 x1862022294232576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767770.802401:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767770.802402:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767770.802407:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767770.802409:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767770.802414:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fc000 x1862022294232576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767770.802422:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767770.802428:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fc000 x1862022294232576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767792 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767770.802436:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767770.802439:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767770.802441:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767770.802442:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767770.802446:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767770.802452:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767770.802457:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767770.802463:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767770.802466:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767770.802471:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fc000 x1862022294232576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767792 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767770.802480:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294232576:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767770.802485:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767770.802489:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767767/real 1775767770] req@ffff9942099ff800 x1862022294232448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767770.802498:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767770.802499:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767770.802501:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39318. 00000400:00000200:3.0:1775767770.802503:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f330 00000400:00000010:3.0:1775767770.802505:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f330. 00000100:00000001:3.0:1775767770.802507:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767770.802511:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ff800 x1862022294232448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767770.802527:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ff800 x1862022294232448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767770.802534:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767770.802535:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767770.802537:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767770.802539:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767770.802543:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ff800 x1862022294232448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767770.802550:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767770.802554:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ff800 x1862022294232448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767792 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767770.802562:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767770.802565:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767770.802567:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767770.802568:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767770.802570:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767770.802573:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767770.802576:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767770.802580:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767770.802583:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767770.802588:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ff800 x1862022294232448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767792 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767770.802597:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294232448:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767770.802602:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767770.802606:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767767/real 1775767770] req@ffff9942099fd500 x1862022294232320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767770.802621:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767770.802623:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767770.802625:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b391b8. 00000400:00000200:3.0:1775767770.802628:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f770 00000400:00000010:3.0:1775767770.802631:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f770. 00000100:00000001:3.0:1775767770.802633:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767770.802637:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fd500 x1862022294232320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767770.802646:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fd500 x1862022294232320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767770.802653:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767770.802655:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767770.802656:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767770.802657:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767770.802660:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fd500 x1862022294232320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767770.802665:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767770.802668:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fd500 x1862022294232320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767792 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767770.802672:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767770.802673:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767770.802674:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767770.802675:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767770.802677:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767770.802678:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767770.802680:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767770.802683:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767770.802685:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767770.802687:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fd500 x1862022294232320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767792 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767770.802694:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294232320:192.168.202.156@tcp:250:kworker.0 00000100:00000001:3.0:1775767770.802697:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767770.802699:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767770.802701:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fc000 x1862022294232576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767792 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767770.802706:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767770.802710:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767770.802715:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f000. 02000000:00000001:3.0:1775767770.802719:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767770.802721:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767770.802723:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:3.0:1775767770.802724:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767770.802727:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002b800. 02000000:00000001:3.0:1775767770.802731:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767770.802735:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767770.802737:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fc000. 00000100:00000001:3.0:1775767770.802742:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767770.802744:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767770.802745:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767770.802749:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ff800 x1862022294232448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767792 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767770.802755:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767770.802757:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767770.802759:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1e800. 02000000:00000001:3.0:1775767770.802762:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767770.802764:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767770.802765:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:3.0:1775767770.802767:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767770.802769:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029800. 02000000:00000001:3.0:1775767770.802772:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767770.802773:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767770.802774:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ff800. 00000100:00000001:3.0:1775767770.802779:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767770.802780:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767770.802782:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767770.802786:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fd500 x1862022294232320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767792 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767770.802794:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767770.802795:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767770.802797:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1ec00. 02000000:00000001:3.0:1775767770.802799:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767770.802801:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767770.802802:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:3.0:1775767770.802804:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767770.802805:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002a800. 02000000:00000001:3.0:1775767770.802807:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767770.802808:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767770.802809:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fd500. 00000100:00000001:3.0:1775767770.802811:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767770.802812:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767770.802815:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767770.802816:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767770.802817:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767770.802818:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767770.802820:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767770.802900:0:2387:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775767770.802905:0:2387:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775767770.802911:0:2387:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775767770.802913:0:2387:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775767770.802917:0:2387:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767770.802922:0:2387:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9942099ff100 x1862022294232704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767792 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767770.802945:0:2387:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767770.802949:0:2387:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ff100 x1862022294232704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767792 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767770.802957:0:2387:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767770.802958:0:2387:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1775767770.802967:0:2387:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000800:00000010:2.0:1775767770.802970:0:2387:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3300. 00000400:00000200:2.0:1775767770.802972:0:2387:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775767770.802975:0:2387:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775767770.802977:0:2387:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f220 00000400:00000010:2.0:1775767770.802979:0:2387:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f220. 00000100:00000001:2.0:1775767770.802980:0:2387:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767770.802984:0:2387:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9942099fc700 x1862022294232832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767792 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767770.802991:0:2387:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767770.802994:0:2387:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fc700 x1862022294232832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767792 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767770.803003:0:2387:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767770.803005:0:2387:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767770.803230:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767770.803232:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767770.803234:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767770.803239:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767767/real 1775767770] req@ffff9942099fc700 x1862022294232832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767770.803246:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767770.803247:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767770.803249:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39e70. 00000400:00000200:3.0:1775767770.803252:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8ff68 00000400:00000010:3.0:1775767770.803253:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8ff68. 00000100:00000001:3.0:1775767770.803255:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767770.803257:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fc700 x1862022294232832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767770.803267:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fc700 x1862022294232832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767770.803272:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767770.803273:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767770.803274:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767770.803275:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767770.803277:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fc700 x1862022294232832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767770.803281:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767770.803283:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fc700 x1862022294232832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767792 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767770.803287:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767770.803289:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767770.803290:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767770.803291:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767770.803292:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767770.803295:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767770.803297:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767770.803299:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767770.803301:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767770.803304:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fc700 x1862022294232832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767792 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767770.803309:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294232832:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767770.803312:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767770.803314:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767767/real 1775767770] req@ffff9942099ff100 x1862022294232704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767770.803323:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767770.803324:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767770.803326:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39630. 00000400:00000200:3.0:1775767770.803327:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f440 00000400:00000010:3.0:1775767770.803328:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f440. 00000100:00000001:3.0:1775767770.803329:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767770.803331:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ff100 x1862022294232704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767770.803336:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ff100 x1862022294232704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767770.803340:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767770.803340:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767770.803341:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767770.803342:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767770.803344:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ff100 x1862022294232704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767770.803348:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767770.803350:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ff100 x1862022294232704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767792 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767770.803354:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767770.803355:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767770.803356:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767770.803357:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767770.803358:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767770.803361:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767770.803362:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767770.803365:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767770.803366:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767770.803369:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ff100 x1862022294232704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767792 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767770.803374:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294232704:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767770.803377:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767770.803378:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767770.803380:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fc700 x1862022294232832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767792 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767770.803384:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767770.803385:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767770.803386:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1c800. 02000000:00000001:3.0:1775767770.803388:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767770.803389:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767770.803390:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:3.0:1775767770.803391:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767770.803393:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029400. 02000000:00000001:3.0:1775767770.803395:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767770.803396:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767770.803397:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fc700. 00000100:00000001:3.0:1775767770.803399:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767770.803400:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767770.803402:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767770.803405:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ff100 x1862022294232704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767792 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767770.803413:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767770.803414:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767770.803416:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f400. 02000000:00000001:3.0:1775767770.803418:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767770.803419:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767770.803421:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:3.0:1775767770.803422:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767770.803424:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002bc00. 02000000:00000001:3.0:1775767770.803426:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767770.803427:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767770.803429:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ff100. 00000100:00000001:3.0:1775767770.803431:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767770.803432:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767770.803434:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767770.803444:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767770.803447:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767770.803449:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767770.803450:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767770.865371:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767770.865376:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767770.865380:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775767771.825247:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767771.825247:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767771.825253:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767771.825254:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767771.825255:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767771.825260:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767771.825263:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767771.825264:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767771.825267:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767771.825269:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775767771.825270:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767771.825273:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767771.825275:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767771.825277:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767771.825279:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767771.825283:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767771.825285:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767771.825287:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767771.825303:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767771.825306:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767771.825308:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767771.825310:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767771.825312:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767771.825315:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767771.825317:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767771.825318:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767771.825320:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767772.849302:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767772.849314:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767772.849316:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767772.849327:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767772.849332:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767772.849334:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767772.849337:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767772.849340:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775767772.849340:0:9316:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775767772.849341:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767772.849353:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00100000:1.0:1775767772.849354:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00080000:3.0:1775767772.849357:0:9316:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775767772.849359:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767772.849360:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767772.849360:0:9316:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1775767772.849364:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767772.849369:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775767772.849371:0:9316:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775767772.849373:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767772.849378:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767772.849379:0:9316:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775767772.849387:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767772.849392:0:9316:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5025 00000400:00000200:3.0:1775767772.849419:0:9316:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767772.849431:0:9316:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767772.849436:0:9316:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767772.849440:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767772.849445:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767772.849448:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767772.849450:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767772.849454:0:9316:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767772.849457:0:9316:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:3.0:1775767772.849460:0:9316:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767772.849464:0:9316:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:3.0:1775767772.849467:0:9316:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:3.0:1775767772.849470:0:9316:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:3.0:1775767772.849473:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767772.849475:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767772.849477:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767772.849480:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767772.849484:0:9316:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:3.0:1775767772.849488:0:9316:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767772.849491:0:9316:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767772.849495:0:9316:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767772.849500:0:9316:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767772.849504:0:9316:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767772.849507:0:9316:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775767772.849510:0:9316:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767772.849520:0:9316:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8e00. 00000020:00000040:3.0:1775767772.849524:0:9316:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:3.0:1775767772.849528:0:9316:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767772.849530:0:9316:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767772.849535:0:9316:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767772.849538:0:9316:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767772.849544:0:9316:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767772.849556:0:9316:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b1000. 00000100:00000001:3.0:1775767772.849564:0:9316:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767772.849576:0:9316:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8e00 x1862022294232960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767772.849588:0:9316:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767772.849595:0:9316:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767772.849601:0:9316:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000864a6866] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8e00 x1862022294232960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767772.849632:0:9316:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767772.849636:0:9316:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767772.849640:0:9316:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775767772.849641:0:9316:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767772.849643:0:9316:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767772.849646:0:9316:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767772.849649:0:9316:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5025 00000400:00000200:3.0:1775767772.849653:0:9316:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767772.849657:0:9316:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767772.849659:0:9316:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767772.849663:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767772.849666:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767772.849669:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767772.849672:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767772.849675:0:9316:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767772.849677:0:9316:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:3.0:1775767772.849679:0:9316:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767772.849682:0:9316:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:3.0:1775767772.849684:0:9316:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:3.0:1775767772.849687:0:9316:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000001:2.0:1775767772.849688:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1775767772.849691:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767772.849694:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775767772.849695:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1775767772.849696:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767772.849700:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1775767772.849700:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767772.849705:0:9316:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00080000:3.0:1775767772.849710:0:9316:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000040:2.0:1775767772.849711:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a8e00 x1862022294232960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767772.849713:0:9316:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767772.849717:0:9316:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1775767772.849725:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1775767772.849727:0:9316:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767772.849728:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767772.849734:0:9316:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767772.849736:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1775767772.849737:0:9316:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:2.0:1775767772.849741:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00100000:3.0:1775767772.849744:0:9316:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1775767772.849746:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1775767772.849752:0:9316:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767772.849754:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294232960:192.168.202.156@tcp:250:kworker.0 00000100:00000001:3.0:1775767772.849756:0:9316:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767772.849763:0:9316:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ab480. 00000100:00000001:2.0:1775767772.849768:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000040:3.0:1775767772.849768:0:9316:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:3.0:1775767772.849773:0:9316:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775767772.849775:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767772.849775:0:9316:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767772.849778:0:9316:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775767772.849779:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767772.849780:0:9316:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767772.849784:0:9316:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767772.849786:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767772.849790:0:9316:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2000. 00000100:00000001:3.0:1775767772.849795:0:9316:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775767772.849796:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1fc00. 02000000:00000001:2.0:1775767772.849800:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767772.849804:0:9316:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ab480 x1862022294233088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1775767772.849812:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767772.849820:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39c60. 00000100:00000001:3.0:1775767772.849821:0:9316:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767772.849826:0:9316:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767772.849830:0:9316:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000067a0ba74] to pc [ptlrpcd_rcv+-1] req@ffff9942099ab480 x1862022294233088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1775767772.849831:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f220. 00000100:00000200:2.0:1775767772.849843:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294232960, portal 25 00000100:00000001:3.0:1775767772.849843:0:9316:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767772.849848:0:9316:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775767772.849849:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767772.849852:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640856064 : -112966068695552 : ffff9942099a8e00) 00000100:00080000:3.0:1775767772.849856:0:9316:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775767772.849859:0:9316:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767772.849861:0:9316:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:2.0:1775767772.849863:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a8e00 x1862022294232960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767797 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767772.849867:0:9316:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767772.849874:0:9316:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5025 00000100:00000001:2.0:1775767772.849880:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:3.0:1775767772.849882:0:9316:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000040:2.0:1775767772.849887:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767772.849890:0:9316:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000400:00000010:2.0:1775767772.849893:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00080000:3.0:1775767772.849894:0:9316:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767772.849899:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000200:2.0:1775767772.849901:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294232960, offset 0 00000100:00000001:3.0:1775767772.849902:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767772.849907:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767772.849910:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000200:2.0:1775767772.849913:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000020:00000001:3.0:1775767772.849915:0:9316:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767772.849918:0:9316:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:3.0:1775767772.849920:0:9316:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767772.849922:0:9316:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:3.0:1775767772.849925:0:9316:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:3.0:1775767772.849927:0:9316:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000400:00000200:2.0:1775767772.849929:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000040:3.0:1775767772.849935:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767772.849937:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767772.849939:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000200:2.0:1775767772.849943:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00000001:3.0:1775767772.849944:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767772.849950:0:9316:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:3.0:1775767772.849954:0:9316:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000400:00000200:2.0:1775767772.849955:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1775767772.849959:0:9316:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767772.849963:0:9316:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:2.0:1775767772.849965:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 02000000:00000001:3.0:1775767772.849968:0:9316:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767772.849971:0:9316:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767772.849972:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1775767772.849975:0:9316:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767772.849978:0:9316:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775767772.849981:0:9316:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767772.849983:0:9316:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767772.849985:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00000010:3.0:1775767772.849989:0:9316:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a9180. 00000020:00000040:3.0:1775767772.849993:0:9316:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000400:00000200:2.0:1775767772.849995:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224993:224993:256:4294967295] 192.168.202.156@tcp LPNI seq info [224989:224989:8:4294967295] 02000000:00000001:3.0:1775767772.849999:0:9316:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767772.850002:0:9316:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767772.850005:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767772.850006:0:9316:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767772.850009:0:9316:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767772.850011:0:9316:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767772.850017:0:9316:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b3400. 00000100:00000001:3.0:1775767772.850023:0:9316:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767772.850024:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000100:00100000:3.0:1775767772.850033:0:9316:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a9180 x1862022294233216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:2.0:1775767772.850035:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00000001:3.0:1775767772.850055:0:9316:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767772.850058:0:9316:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767772.850063:0:9316:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bdc2586] to pc [ptlrpcd_rcv+-1] req@ffff9942099a9180 x1862022294233216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000010:2.0:1775767772.850069:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3300. 00000100:00000001:3.0:1775767772.850075:0:9316:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767772.850079:0:9316:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:2.0:1775767772.850086:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00080000:3.0:1775767772.850087:0:9316:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000200:2.0:1775767772.850107:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775767772.850113:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:3.0:1775767772.850118:0:9316:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000800:00000200:2.1:1775767772.850119:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000100:00080000:3.0:1775767772.850122:0:9316:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1775767772.850128:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000100:00000001:3.0:1775767772.850128:0:9316:0:(import.c:484:import_select_connection()) Process entered 00000800:00000010:2.0:1775767772.850132:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3300. 00000100:00080000:3.0:1775767772.850134:0:9316:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5025 00000400:00000200:2.0:1775767772.850141:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767772.850141:0:9316:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767772.850147:0:9316:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000400:00000200:2.0:1775767772.850152:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000100:00080000:3.0:1775767772.850153:0:9316:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767772.850187:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767772.850190:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767772.850193:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767772.850198:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767772.850201:0:9316:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767772.850202:0:9316:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000400:00000200:2.0:1775767772.850203:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000020:00000001:3.0:1775767772.850204:0:9316:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000010:2.0:1775767772.850207:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000020:00000040:3.0:1775767772.850208:0:9316:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:3.0:1775767772.850213:0:9316:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:2.0:1775767772.850216:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:3.0:1775767772.850216:0:9316:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:3.0:1775767772.850222:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000200:2.0:1775767772.850226:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a8e00 x1862022294232960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767797 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767772.850227:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767772.850234:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767772.850239:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:2.0:1775767772.850244:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000020:00000040:3.0:1775767772.850246:0:9316:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00000040:2.0:1775767772.850252:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a8e00 x1862022294232960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767797 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767772.850252:0:9316:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767772.850258:0:9316:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767772.850264:0:9316:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1775767772.850269:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767772.850269:0:9316:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767772.850273:0:2399:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:3.0:1775767772.850275:0:9316:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767772.850278:0:9316:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775767772.850282:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1775767772.850282:0:9316:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:3.0:1775767772.850286:0:9316:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767772.850290:0:9316:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767772.850294:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000010:3.0:1775767772.850298:0:9316:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8700. 00000100:00000001:2.0:1775767772.850299:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299699006 00000020:00000040:3.0:1775767772.850304:0:9316:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00000100:00000010:2.0:1775767772.850308:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d6780. 02000000:00000001:3.0:1775767772.850309:0:9316:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767772.850312:0:9316:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767772.850316:0:9316:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767772.850317:0:9316:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767772.850319:0:9316:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767772.850322:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 02000000:00000010:3.0:1775767772.850325:0:9316:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0400. 00000100:00000001:3.0:1775767772.850331:0:9316:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767772.850333:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299699006 00000100:00100000:3.0:1775767772.850338:0:9316:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8700 x1862022294233344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000010:2.0:1775767772.850340:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d60c0. 00000100:00000001:2.0:1775767772.850347:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1775767772.850351:0:9316:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775767772.850353:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767772.850354:0:9316:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767772.850359:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767772.850362:0:9316:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000faa25e3b] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8700 x1862022294233344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767772.850365:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767772.850374:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767772/real 1775767772] req@ffff9942099a8e00 x1862022294232960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767772.850375:0:9316:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767772.850381:0:9316:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767772.850390:0:9316:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775767772.850394:0:9316:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767772.850396:0:9316:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000001:2.0:1775767772.850397:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767772.850400:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767772.850403:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39c60. 00000400:00000200:2.0:1775767772.850408:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f220 00000400:00000010:2.0:1775767772.850409:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f220. 00000100:00000001:2.0:1775767772.850413:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767772.850418:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a8e00 x1862022294232960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767772.850429:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a8e00 x1862022294232960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767772.850438:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767772.850440:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767772.850443:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767772.850445:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767772.850449:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a8e00 x1862022294232960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767772.850458:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767772.850463:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a8e00 x1862022294232960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767797 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767772.850471:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767772.850475:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767772.850477:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767772.850478:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767772.850482:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767772.850485:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767772.850488:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767772.850494:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767772.850496:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767772.850501:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a8e00 x1862022294232960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767797 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767772.850509:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294232960:192.168.202.156@tcp:250:kworker.0 00000100:00000001:2.0:1775767772.850518:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767772.850521:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767772.850524:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a8e00 x1862022294232960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767797 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767772.850531:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767772.850535:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767772.850538:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1fc00. 02000000:00000001:2.0:1775767772.850542:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767772.850546:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767772.850547:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:2.0:1775767772.850550:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767772.850553:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b1000. 02000000:00000001:2.0:1775767772.850558:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767772.850561:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767772.850571:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a8e00. 00000100:00000001:2.0:1775767772.850575:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767772.850576:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767772.850578:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767772.850590:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767772.850592:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767772.850594:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775767772.850596:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767772.850598:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767772.850601:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ab480 x1862022294233088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767772.850609:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767772.850610:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767772.850613:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767772.850615:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767772.850618:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767772.850626:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ab480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294233088:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775767772.850630:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767772.850633:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767772.850635:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767772.850637:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775767772.850641:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1fc00. 02000000:00000001:2.0:1775767772.850643:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767772.850647:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767772.850650:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39c60. 00000400:00000010:2.0:1775767772.850655:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f220. 00000100:00000200:2.0:1775767772.850659:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294233088, portal 10 00000100:00000001:2.0:1775767772.850661:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767772.850663:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640865920 : -112966068685696 : ffff9942099ab480) 00000100:00000040:2.0:1775767772.850677:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ab480 x1862022294233088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767797 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767772.850684:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767772.850687:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767772.850690:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775767772.850694:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294233088, offset 0 00000400:00000200:2.0:1775767772.850703:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767772.850712:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767772.850718:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767772.850722:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767772.850725:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767772.850728:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767772.850731:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00000001:3.0:1775767772.850731:0:9316:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:2.0:1775767772.850737:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224994:224994:256:4294967295] 192.168.202.156@tcp LPNI seq info [224990:224990:8:4294967295] 00000100:00080000:3.0:1775767772.850740:0:9316:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5025 00000400:00000200:2.0:1775767772.850746:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767772.850753:0:9316:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767772.850759:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000400:00000200:3.0:1775767772.850763:0:9316:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767772.850767:0:9316:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000800:00000200:2.0:1775767772.850772:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00000040:3.0:1775767772.850776:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000800:00000010:2.0:1775767772.850778:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3300. 00000100:00000001:3.0:1775767772.850780:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767772.850784:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000800:00000200:2.0:1775767772.850788:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775767772.850791:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775767772.850797:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767772.850802:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000100:00000001:3.0:1775767772.850805:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000800:00000100:2.0:1775767772.850807:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767772.850810:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3300. 00000020:00000001:3.0:1775767772.850811:0:9316:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767772.850813:0:9316:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000400:00000200:2.0:1775767772.850817:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000001:3.0:1775767772.850817:0:9316:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767772.850819:0:9316:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:3.0:1775767772.850824:0:9316:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000400:00000200:2.0:1775767772.850825:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000020:00000001:3.0:1775767772.850830:0:9316:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000400:00000200:2.0:1775767772.850832:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775767772.850835:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000040:3.0:1775767772.850837:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767772.850839:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1775767772.850842:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:2.0:1775767772.850846:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ab480 x1862022294233088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767797 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775767772.850848:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767772.850854:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767772.850860:0:9316:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00000001:2.0:1775767772.850864:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00080000:3.0:1775767772.850867:0:9316:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000040:2.0:1775767772.850871:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ab480 x1862022294233088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767797 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767772.850873:0:9316:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767772.850878:0:9316:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1775767772.850884:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767772.850885:0:9316:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767772.850888:0:2399:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:3.0:1775767772.850890:0:9316:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767772.850892:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767772.850894:0:9316:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767772.850897:0:9316:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000100:00000040:2.0:1775767772.850899:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000008:00000001:3.0:1775767772.850902:0:9316:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767772.850906:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767772.850906:0:9316:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767772.850911:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299697683 00000100:00000010:3.0:1775767772.850914:0:9316:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8380. 00000100:00000001:2.0:1775767772.850915:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299699006 00000020:00000040:3.0:1775767772.850919:0:9316:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00000100:00000001:2.0:1775767772.850920:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299697683 00000100:00000001:2.0:1775767772.850926:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299699006 02000000:00000001:3.0:1775767772.850926:0:9316:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1775767772.850930:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767772.850931:0:9316:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767772.850936:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767772.850936:0:9316:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767772.850940:0:9316:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1775767772.850941:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 02000000:00000001:3.0:1775767772.850944:0:9316:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767772.850947:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767772/real 1775767772] req@ffff9942099ab480 x1862022294233088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1775767772.850951:0:9316:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0000. 00000100:00000001:3.0:1775767772.850959:0:9316:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1775767772.850961:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767772.850964:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767772.850967:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39c60. 00000100:00100000:3.0:1775767772.850968:0:9316:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8380 x1862022294233472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1775767772.850971:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f220 00000100:00000001:3.0:1775767772.850978:0:9316:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000010:2.0:1775767772.850981:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f220. 00000100:00000001:3.0:1775767772.850984:0:9316:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767772.850986:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:3.0:1775767772.850991:0:9316:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055f5621a] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8380 x1862022294233472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767772.850993:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ab480 x1862022294233088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767772.851027:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ab480 x1862022294233088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767772.851028:0:9316:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767772.851038:0:9316:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5035) 00000100:00000001:2.0:1775767772.851039:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767772.851043:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767772.851045:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767772.851047:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767772.851051:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ab480 x1862022294233088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767772.851059:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767772.851063:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ab480 x1862022294233088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767797 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767772.851072:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767772.851075:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767772.851077:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767772.851079:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767772.851081:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767772.851085:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767772.851088:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767772.851105:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767772.851108:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767772.851112:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ab480 x1862022294233088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767797 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767772.851121:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ab480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294233088:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775767772.851126:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767772.851131:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a9180 x1862022294233216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767772.851145:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767772.851147:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767772.851150:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767772.851152:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767772.851185:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767772.851190:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294233216:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775767772.851194:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767772.851197:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767772.851199:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767772.851201:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775767772.851205:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1d400. 02000000:00000001:2.0:1775767772.851207:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767772.851211:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767772.851214:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39c60. 00000400:00000010:2.0:1775767772.851218:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f220. 00000100:00000200:2.0:1775767772.851221:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294233216, portal 10 00000100:00000001:2.0:1775767772.851223:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767772.851224:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640856960 : -112966068694656 : ffff9942099a9180) 00000100:00000040:2.0:1775767772.851230:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a9180 x1862022294233216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767797 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767772.851237:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767772.851240:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767772.851242:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775767772.851245:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294233216, offset 0 00000400:00000200:2.0:1775767772.851249:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767772.851254:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767772.851258:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767772.851261:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767772.851264:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767772.851266:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767772.851268:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767772.851272:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224995:224995:256:4294967295] 192.168.202.156@tcp LPNI seq info [224991:224991:8:4294967295] 00000400:00000200:2.0:1775767772.851277:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767772.851285:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767772.851290:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767772.851294:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3300. 00000800:00000200:2.0:1775767772.851298:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775767772.851301:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775767772.851306:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767772.851309:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1775767772.851313:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767772.851315:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3300. 00000400:00000200:2.0:1775767772.851319:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775767772.851322:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775767772.851326:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775767772.851328:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775767772.851331:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767772.851334:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a9180 x1862022294233216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767797 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767772.851343:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767772.851346:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a9180 x1862022294233216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767797 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767772.851353:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767772.851354:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775767772.851358:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767772.851361:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:2.0:1775767772.851365:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767772.851367:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299697676 00000100:00000001:2.0:1775767772.851370:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299699006 00000100:00000001:2.0:1775767772.851372:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299697676 00000100:00000001:2.0:1775767772.851375:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299699006 00000100:00000001:2.0:1775767772.851377:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767772.851380:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767772.851382:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767772.851386:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767772/real 1775767772] req@ffff9942099a9180 x1862022294233216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775767772.851395:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767772.851397:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767772.851399:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39c60. 00000400:00000200:2.0:1775767772.851401:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f220 00000400:00000010:2.0:1775767772.851403:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f220. 00000100:00000001:2.0:1775767772.851405:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767772.851409:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a9180 x1862022294233216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767772.851417:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a9180 x1862022294233216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767772.851424:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767772.851426:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767772.851427:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767772.851429:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767772.851432:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a9180 x1862022294233216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767772.851438:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767772.851443:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a9180 x1862022294233216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767797 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767772.851451:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767772.851454:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767772.851455:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767772.851458:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767772.851459:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767772.851463:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767772.851466:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767772.851470:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767772.851472:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767772.851477:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a9180 x1862022294233216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767797 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767772.851486:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294233216:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775767772.851491:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767772.851495:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a8700 x1862022294233344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767772.851503:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767772.851505:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767772.851507:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767772.851509:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767772.851512:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767772.851515:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294233344:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767772.851517:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767772.851519:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767772.851520:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767772.851530:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775767772.851534:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f800. 02000000:00000001:2.0:1775767772.851536:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767772.851540:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767772.851543:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39c60. 00000400:00000010:2.0:1775767772.851546:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f220. 00000100:00000200:2.0:1775767772.851550:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294233344, portal 4 00000100:00000001:2.0:1775767772.851552:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767772.851553:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640854272 : -112966068697344 : ffff9942099a8700) 00000100:00000040:2.0:1775767772.851560:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a8700 x1862022294233344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767797 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767772.851568:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767772.851572:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767772.851574:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775767772.851577:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294233344, offset 0 00000400:00000200:2.0:1775767772.851582:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767772.851586:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767772.851590:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767772.851593:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767772.851596:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767772.851599:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767772.851601:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767772.851604:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224996:224996:256:4294967295] 192.168.202.156@tcp LPNI seq info [224992:224992:8:4294967295] 00000400:00000200:2.0:1775767772.851609:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767772.851617:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767772.851622:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767772.851626:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3300. 00000800:00000200:2.0:1775767772.851630:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775767772.851634:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775767772.851638:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767772.851641:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1775767772.851645:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767772.851647:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3300. 00000400:00000200:2.0:1775767772.851650:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775767772.851654:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775767772.851657:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775767772.851659:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775767772.851661:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767772.851665:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a8700 x1862022294233344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767797 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767772.851675:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767772.851678:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a8700 x1862022294233344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767797 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767772.851684:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767772.851686:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775767772.851689:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767772.851694:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:2.0:1775767772.851697:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767772.851700:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299697699 00000100:00000001:2.0:1775767772.851702:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299699006 00000100:00000001:2.0:1775767772.851705:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299697699 00000100:00000001:2.0:1775767772.851707:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299699006 00000100:00000001:2.0:1775767772.851709:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767772.851712:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767772.851714:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767772.851718:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767772/real 1775767772] req@ffff9942099a8700 x1862022294233344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775767772.851728:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767772.851729:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767772.851731:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39c60. 00000400:00000200:2.0:1775767772.851734:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f220 00000400:00000010:2.0:1775767772.851736:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f220. 00000100:00000001:2.0:1775767772.851738:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767772.851741:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a8700 x1862022294233344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767772.851750:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a8700 x1862022294233344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767772.851757:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767772.851759:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767772.851762:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767772.851763:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767772.851767:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a8700 x1862022294233344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767772.851774:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767772.851778:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a8700 x1862022294233344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767797 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767772.851787:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767772.851790:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767772.851792:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767772.851794:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767772.851796:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767772.851801:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767772.851803:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767772.851808:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767772.851811:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767772.851815:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a8700 x1862022294233344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767797 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767772.851824:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294233344:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767772.851828:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767772.851830:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767772.851833:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ab480 x1862022294233088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767797 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767772.851841:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767772.851843:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767772.851846:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1fc00. 02000000:00000001:2.0:1775767772.851850:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767772.851852:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767772.851854:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:2.0:1775767772.851856:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767772.851858:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b2000. 02000000:00000001:2.0:1775767772.851862:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767772.851864:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767772.851866:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ab480. 00000100:00000001:2.0:1775767772.851871:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767772.851872:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767772.851874:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767772.851877:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a9180 x1862022294233216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767797 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767772.851885:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767772.851886:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767772.851888:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1d400. 02000000:00000001:2.0:1775767772.851892:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767772.851893:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767772.851895:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:2.0:1775767772.851897:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767772.851898:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b3400. 02000000:00000001:2.0:1775767772.851901:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767772.851902:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767772.851904:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a9180. 00000100:00000001:2.0:1775767772.851906:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767772.851907:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767772.851909:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767772.851912:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a8700 x1862022294233344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767797 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767772.851919:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767772.851921:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767772.851922:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f800. 02000000:00000001:2.0:1775767772.851925:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767772.851926:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767772.851927:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:2.0:1775767772.851930:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767772.851931:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b0400. 02000000:00000001:2.0:1775767772.851935:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767772.851936:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767772.851937:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a8700. 00000100:00000001:2.0:1775767772.851939:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767772.851941:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767772.851943:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767772.851950:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767772.851952:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767772.851955:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775767772.851959:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767772.851960:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767772.851965:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a8380 x1862022294233472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767772.851973:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767772.851974:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767772.851977:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767772.851979:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767772.851981:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767772.851985:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294233472:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767772.851989:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767772.851991:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767772.851993:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767772.851995:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775767772.851998:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f800. 02000000:00000001:2.0:1775767772.852000:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767772.852004:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767772.852006:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39c60. 00000400:00000010:2.0:1775767772.852011:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f220. 00000100:00000200:2.0:1775767772.852015:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294233472, portal 4 00000100:00000001:2.0:1775767772.852017:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767772.852018:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640853376 : -112966068698240 : ffff9942099a8380) 00000100:00000040:2.0:1775767772.852024:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a8380 x1862022294233472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767797 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767772.852031:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767772.852033:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767772.852036:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775767772.852039:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294233472, offset 0 00000400:00000200:2.0:1775767772.852043:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767772.852048:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767772.852052:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767772.852055:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767772.852058:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767772.852061:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767772.852064:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767772.852067:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224997:224997:256:4294967295] 192.168.202.156@tcp LPNI seq info [224993:224993:8:4294967295] 00000400:00000200:2.0:1775767772.852071:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767772.852079:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767772.852084:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767772.852088:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3300. 00000800:00000200:2.0:1775767772.852107:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775767772.852110:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775767772.852115:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767772.852119:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1775767772.852123:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767772.852125:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3300. 00000400:00000200:2.0:1775767772.852128:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775767772.852132:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775767772.852135:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775767772.852137:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775767772.852140:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767772.852143:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a8380 x1862022294233472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767797 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767772.852152:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767772.852192:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a8380 x1862022294233472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767797 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767772.852200:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767772.852201:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775767772.852206:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767772.852210:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:2.0:1775767772.852213:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767772.852216:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299697703 00000100:00000001:2.0:1775767772.852218:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299699006 00000100:00000001:2.0:1775767772.852221:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299697703 00000100:00000001:2.0:1775767772.852225:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299699006 00000100:00000001:2.0:1775767772.852227:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767772.852229:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767772.852231:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767772.852236:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767772/real 1775767772] req@ffff9942099a8380 x1862022294233472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775767772.852244:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767772.852246:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767772.852247:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39c60. 00000400:00000200:2.0:1775767772.852250:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f220 00000400:00000010:2.0:1775767772.852251:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f220. 00000100:00000001:2.0:1775767772.852253:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767772.852257:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a8380 x1862022294233472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767772.852265:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a8380 x1862022294233472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767772.852271:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767772.852273:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767772.852275:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767772.852276:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767772.852280:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a8380 x1862022294233472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767772.852287:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767772.852292:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a8380 x1862022294233472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767797 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767772.852300:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767772.852303:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767772.852305:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767772.852306:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767772.852309:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767772.852314:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767772.852317:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767772.852322:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767772.852324:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767772.852329:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a8380 x1862022294233472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767797 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767772.852337:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294233472:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767772.852342:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767772.852344:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767772.852347:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a8380 x1862022294233472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767797 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767772.852355:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767772.852357:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767772.852359:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f800. 02000000:00000001:2.0:1775767772.852363:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767772.852364:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767772.852365:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:2.0:1775767772.852367:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767772.852369:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b0000. 02000000:00000001:2.0:1775767772.852373:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767772.852374:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767772.852376:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a8380. 00000100:00000001:2.0:1775767772.852379:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767772.852380:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767772.852382:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767772.852387:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767772.852389:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767772.852391:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775767772.852393:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767772.913380:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767772.913383:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767772.913387:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775767773.873284:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767773.873292:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767773.873297:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767773.873309:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767773.873317:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767773.873323:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767773.873327:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767773.873330:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767773.873334:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767774.193267:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299699006, at 4299699008 00000100:00000010:1.0:1775767774.193283:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d60c0. 00000100:00000001:1.0:1775767774.897281:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767774.897281:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767774.897289:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767774.897289:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767774.897293:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767774.897293:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767774.897299:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767774.897305:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1775767774.897306:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767774.897310:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767774.897310:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767774.897314:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767774.897321:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767774.897324:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767774.897328:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767774.897328:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767774.897330:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767774.897332:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767774.961324:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767774.961327:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767774.961331:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775767775.921242:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767775.921247:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767775.921249:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767775.921254:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767775.921260:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767775.921264:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767775.921266:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767775.921269:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767775.921271:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767775.921310:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767775.921314:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767775.921316:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767775.921319:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767775.921321:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767775.921321:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767775.921324:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767775.921327:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767775.921328:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767775.921329:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767775.921330:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767775.921334:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767775.921336:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775767775.921341:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767775.921343:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767775.921345:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767775.921348:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767775.921350:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767776.945242:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767776.945248:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767776.945251:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767776.945257:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767776.945261:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767776.945264:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767776.945266:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767776.945269:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767776.945271:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767776.945287:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767776.945292:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767776.945296:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767776.945301:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767776.945306:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767776.945309:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767776.945310:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767776.945312:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767776.945318:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767776.945319:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00100000:1.0:1775767776.945323:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775767776.945323:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767776.945332:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767776.945333:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767776.945340:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767776.945344:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767776.945347:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767776.945349:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767777.009321:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767777.009324:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767777.009326:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775767777.969241:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767777.969248:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767777.969248:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:2.0:1775767777.969250:0:9316:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1775767777.969251:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767777.969253:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767777.969256:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775767777.969259:0:9316:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:1.0:1775767777.969260:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775767777.969260:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767777.969263:0:9316:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775767777.969267:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1775767777.969267:0:9316:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767777.969268:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767777.969272:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767777.969274:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767777.969275:0:9316:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775767777.969276:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767777.969281:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1775767777.969283:0:9316:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5030 00000100:00000001:1.0:1775767777.969287:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767777.969288:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767777.969293:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1775767777.969295:0:9316:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767777.969299:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767777.969306:0:9316:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775767777.969312:0:9316:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000080:00000001:1.0:1775767777.969325:0:36:0:(rw.c:1481:ll_writepages()) Process entered 00000080:00000001:1.0:1775767777.969328:0:36:0:(file.c:5250:cl_sync_file_range()) Process entered 00000100:00000040:2.0:1775767777.969329:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000001:1.0:1775767777.969331:0:36:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00000001:2.0:1775767777.969335:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000010:1.0:1775767777.969339:0:36:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994206aba240. 00000100:00000040:2.0:1775767777.969339:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767777.969345:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000010:1.0:1775767777.969346:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994201343200. 00000020:00000010:1.0:1775767777.969352:0:36:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994201342e00. 00000020:00000001:2.0:1775767777.969355:0:9316:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000010:1.0:1775767777.969356:0:36:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207cba360. 00000020:00000040:2.0:1775767777.969360:0:9316:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:2.0:1775767777.969363:0:9316:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000008:00000010:1.0:1775767777.969364:0:36:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207e02ca0. 00000020:00000040:2.0:1775767777.969365:0:9316:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00020000:00000010:1.0:1775767777.969369:0:36:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff9942031b3480. 00000020:00000001:2.0:1775767777.969370:0:9316:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00008000:00000010:1.0:1775767777.969374:0:36:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942054ab170. 00000020:00000001:2.0:1775767777.969374:0:9316:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000080:00000010:1.0:1775767777.969381:0:36:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99422030bf18. 00000100:00000040:2.0:1775767777.969381:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767777.969385:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000080:00000010:1.0:1775767777.969386:0:36:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff994205751da8. 00000100:00000040:2.0:1775767777.969390:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000010:1.0:1775767777.969392:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994201343000. 00000100:00000001:2.0:1775767777.969394:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000008:00000010:1.0:1775767777.969397:0:36:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff994210709ce0. 00000020:00000040:2.0:1775767777.969399:0:9316:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00020000:00000010:1.0:1775767777.969402:0:36:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994205743180. 00000100:00080000:2.0:1775767777.969405:0:9316:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00008000:00000010:1.0:1775767777.969406:0:36:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994219fec110. 00000080:00000010:1.0:1775767777.969412:0:36:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942057e2960. 00000100:00000001:2.0:1775767777.969412:0:9316:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775767777.969418:0:36:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107591643720 : -112966117907896 : ffff994206aba248) 02000000:00000001:2.0:1775767777.969419:0:9316:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000020:00001000:1.0:1775767777.969421:0:36:0:(cl_object.c:831:cl_env_get()) 1@ffff994206aba240 02000000:00000001:2.0:1775767777.969424:0:9316:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1775767777.969425:0:36:0:(cl_object.c:151:cl_object_top()) ffff99420791a5f8 -> ffff99420791a5f8 02000000:00000001:2.0:1775767777.969429:0:9316:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775767777.969430:0:36:0:(cl_io.c:125:__cl_io_init()) Process entered 00000100:00000001:2.0:1775767777.969433:0:9316:0:(obd_class.h:890:obd_reconnect()) Process entered 00000080:00000001:1.0:1775767777.969434:0:36:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:1.0:1775767777.969437:0:36:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x10176:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000100:00000001:2.0:1775767777.969437:0:9316:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775767777.969443:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:1.0:1775767777.969444:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:1.0:1775767777.969445:0:36:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775767777.969446:0:9316:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfce00. 00020000:00000002:1.0:1775767777.969448:0:36:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x10176:0x0]io ffff994205751e08 type 6 ignore/verify layout 1/0 00000020:00000040:2.0:1775767777.969450:0:9316:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 00020000:00000002:1.0:1775767777.969451:0:36:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207dfb6c8) owner 0000000000000000/ffff994201348000 00020000:00000001:1.0:1775767777.969455:0:36:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 02000000:00000001:2.0:1775767777.969455:0:9316:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00020000:00000001:1.0:1775767777.969458:0:36:0:(lov_io.c:495:lov_io_slice_init()) Process entered 02000000:00000001:2.0:1775767777.969458:0:9316:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775767777.969461:0:36:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00020000:00000001:1.0:1775767777.969464:0:36:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767777.969464:0:9316:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00020000:00000001:1.0:1775767777.969466:0:36:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:1.0:1775767777.969468:0:36:0:(lov_io.c:203:lov_io_subio_init()) Process entered 02000000:00000001:2.0:1775767777.969468:0:9316:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00020000:00000001:1.0:1775767777.969472:0:36:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775767777.969473:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:1.0:1775767777.969474:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:1.0:1775767777.969475:0:36:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 02000000:00000001:2.0:1775767777.969475:0:9316:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:1.0:1775767777.969477:0:36:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207dfb6c8) owner 0000000000000000/ffff994201348000 00000020:00000001:1.0:1775767777.969481:0:36:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775767777.969482:0:36:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:1.0:1775767777.969483:0:36:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00020000:00000001:1.0:1775767777.969485:0:36:0:(lov_io.c:836:lov_io_iter_init()) Process entered 02000000:00000010:2.0:1775767777.969486:0:9316:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1f800. 00020000:00200000:1.0:1775767777.969487:0:36:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00000002:1.0:1775767777.969492:0:36:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00000100:00000001:2.0:1775767777.969495:0:9316:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775767777.969496:0:36:0:(lov_io.c:155:lov_sub_get()) Process entered 00020000:00000001:1.0:1775767777.969499:0:36:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000020:00000001:1.0:1775767777.969501:0:36:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:1.0:1775767777.969504:0:36:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994206ababd0. 00000100:00100000:2.0:1775767777.969508:0:9316:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfce00 x1862022294233600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000010:1.0:1775767777.969510:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff99420120a200. 00000020:00000010:1.0:1775767777.969517:0:36:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994208a8ec00. 00000020:00000010:1.0:1775767777.969522:0:36:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207cbaae0. 00000100:00000001:2.0:1775767777.969525:0:9316:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000008:00000010:1.0:1775767777.969527:0:36:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207e00000. 00000100:00000001:2.0:1775767777.969529:0:9316:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:1.0:1775767777.969533:0:36:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff9942031b30c0. 00000100:00000040:2.0:1775767777.969535:0:9316:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e5a7debe] to pc [ptlrpcd_rcv+-1] req@ffff994207dfce00 x1862022294233600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00008000:00000010:1.0:1775767777.969539:0:36:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942054a8a98. 00000080:00000010:1.0:1775767777.969545:0:36:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99422030b0a8. 00000080:00000010:1.0:1775767777.969549:0:36:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff994205753d98. 00000020:00000010:1.0:1775767777.969553:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994208a8f600. 00000008:00000010:1.0:1775767777.969556:0:36:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff994210709080. 00020000:00000010:1.0:1775767777.969558:0:36:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff9942057418c0. 00000100:00000001:2.0:1775767777.969558:0:9316:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:1.0:1775767777.969560:0:36:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994219fec368. 00000100:00080000:2.0:1775767777.969562:0:9316:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000080:00000010:1.0:1775767777.969563:0:36:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942057e24b0. 00000020:00000001:1.0:1775767777.969567:0:36:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107591646168 : -112966117905448 : ffff994206ababd8) 00000100:00080000:2.0:1775767777.969568:0:9316:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000020:00001000:1.0:1775767777.969571:0:36:0:(cl_object.c:831:cl_env_get()) 1@ffff994206ababd0 00000100:00000001:2.0:1775767777.969571:0:9316:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000020:00000001:1.0:1775767777.969575:0:36:0:(cl_object.c:151:cl_object_top()) ffff99420791ae90 -> ffff99420791a5f8 00000100:00080000:2.0:1775767777.969575:0:9316:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000020:00000001:1.0:1775767777.969580:0:36:0:(cl_io.c:125:__cl_io_init()) Process entered 00000100:00000001:2.0:1775767777.969581:0:9316:0:(import.c:484:import_select_connection()) Process entered 00000020:00000001:1.0:1775767777.969583:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000100:00080000:2.0:1775767777.969585:0:9316:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5030 00000020:00000001:1.0:1775767777.969586:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:1.0:1775767777.969589:0:36:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775767777.969591:0:36:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767777.969593:0:9316:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:1.0:1775767777.969594:0:36:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107571233256 : -112966138318360 : ffff9942057431e8) 00000400:00000200:2.0:1775767777.969597:0:9316:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000020:00000001:1.0:1775767777.969598:0:36:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000100:00080000:2.0:1775767777.969600:0:9316:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000020:00000001:1.0:1775767777.969601:0:36:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767777.969603:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00020000:00000001:1.0:1775767777.969607:0:36:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767777.969607:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:1.0:1775767777.969611:0:36:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767777.969612:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767777.969616:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775767777.969617:0:36:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:2.0:1775767777.969619:0:9316:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:3.0:1775767777.969621:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00020000:00000001:1.0:1775767777.969622:0:36:0:(lov_io.c:1108:lov_io_lock()) Process entered 00000020:00000040:2.0:1775767777.969622:0:9316:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000100:00000001:3.0:1775767777.969626:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00020000:00000001:1.0:1775767777.969627:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775767777.969627:0:9316:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:3.0:1775767777.969629:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000040:2.0:1775767777.969630:0:9316:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:1.0:1775767777.969631:0:36:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:2.0:1775767777.969634:0:9316:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:1.0:1775767777.969635:0:36:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000100:00000040:3.0:1775767777.969638:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfce00 x1862022294233600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1775767777.969639:0:9316:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:1.0:1775767777.969640:0:36:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775767777.969645:0:36:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767777.969646:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00020000:00000001:1.0:1775767777.969651:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767777.969651:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767777.969656:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00020000:00000001:1.0:1775767777.969659:0:36:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767777.969659:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:3.0:1775767777.969659:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767777.969662:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775767777.969665:0:36:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000040:2.0:1775767777.969665:0:9316:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 02000000:00000001:3.0:1775767777.969666:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:1.0:1775767777.969669:0:36:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767777.969670:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1775767777.969671:0:9316:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000020:00000001:1.0:1775767777.969675:0:36:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767777.969676:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767777.969677:0:9316:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775767777.969680:0:36:0:(cl_io.c:528:cl_io_start()) Process entered 02000000:00000001:2.0:1775767777.969682:0:9316:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00020000:00000001:1.0:1775767777.969684:0:36:0:(lov_io.c:1114:lov_io_start()) Process entered 00000100:00100000:3.0:1775767777.969685:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294233600:192.168.202.156@tcp:250:kworker.0 00020000:00000001:1.0:1775767777.969687:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:1.0:1775767777.969690:0:36:0:(cl_io.c:528:cl_io_start()) Process entered 00000100:00000001:3.0:1775767777.969694:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000008:00000001:1.0:1775767777.969695:0:36:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 02000000:00000001:2.0:1775767777.969695:0:9316:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00000001:1.0:1775767777.969700:0:36:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 02000000:00000001:2.0:1775767777.969700:0:9316:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767777.969700:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1775767777.969702:0:9316:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:3.0:1775767777.969706:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1775767777.969708:0:9316:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775767777.969709:0:36:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 02000000:00000001:3.0:1775767777.969711:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000020:1.0:1775767777.969712:0:36:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:2.0:1775767777.969713:0:9316:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1775767777.969715:0:36:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767777.969716:0:9316:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767777.969718:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b2800. 00000008:00000001:1.0:1775767777.969721:0:36:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000100:00000010:2.0:1775767777.969723:0:9316:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dff480. 02000000:00000001:3.0:1775767777.969723:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1775767777.969724:0:36:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000008:00000001:1.0:1775767777.969727:0:36:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107754940544 : -112965954611072 : ffff994210675880) 00000020:00000040:2.0:1775767777.969729:0:9316:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000008:00000020:1.0:1775767777.969731:0:36:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994210675880 ready 0|-|- wr 1|-|+ rd 0|- 10 in flight 02000000:00000001:2.0:1775767777.969732:0:9316:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775767777.969735:0:9316:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767777.969736:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767777.969739:0:9316:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000008:00000001:1.0:1775767777.969740:0:36:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 02000000:00000001:2.0:1775767777.969741:0:9316:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000008:00000020:1.0:1775767777.969742:0:36:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:1.0:1775767777.969744:0:36:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1775767777.969744:0:9316:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1775767777.969744:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000008:00000001:1.0:1775767777.969750:0:36:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000008:00000020:1.0:1775767777.969753:0:36:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994218232fc0 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 02000000:00000010:2.0:1775767777.969753:0:9316:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1d400. 00000400:00000010:3.0:1775767777.969756:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000001:2.0:1775767777.969758:0:9316:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1775767777.969761:0:36:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:1.0:1775767777.969764:0:36:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767777.969766:0:9316:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dff480 x1862022294233728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000001:1.0:1775767777.969767:0:36:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1775767777.969767:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294233600, portal 25 00020000:00000001:1.0:1775767777.969771:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767777.969775:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00020000:00000001:1.0:1775767777.969776:0:36:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767777.969780:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611840000 : -112966097711616 : ffff994207dfce00) 00000020:00000001:1.0:1775767777.969781:0:36:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775767777.969786:0:36:0:(cl_io.c:555:cl_io_end()) Process entered 00000100:00000001:2.0:1775767777.969788:0:9316:0:(jobid.c:936:lustre_get_jobid()) Process entered 00020000:00000001:1.0:1775767777.969789:0:36:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00000100:00000001:2.0:1775767777.969791:0:9316:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775767777.969792:0:36:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000020:00000001:1.0:1775767777.969795:0:36:0:(cl_io.c:555:cl_io_end()) Process entered 00000100:00000040:3.0:1775767777.969795:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfce00 x1862022294233600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767802 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1775767777.969798:0:36:0:(cl_io.c:563:cl_io_end()) Process leaving 00020000:00000001:1.0:1775767777.969800:0:36:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767777.969800:0:9316:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093a15639] to pc [ptlrpcd_rcv+-1] req@ffff994207dff480 x1862022294233728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:1.0:1775767777.969805:0:36:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000020:00000001:1.0:1775767777.969808:0:36:0:(cl_io.c:563:cl_io_end()) Process leaving 00000020:00000001:1.0:1775767777.969811:0:36:0:(cl_io.c:352:cl_io_unlock()) Process entered 00020000:00000001:1.0:1775767777.969814:0:36:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00020000:00000001:1.0:1775767777.969816:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:1.0:1775767777.969819:0:36:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000020:00000001:1.0:1775767777.969822:0:36:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:1.0:1775767777.969824:0:36:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775767777.969830:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767777.969831:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00020000:00000001:1.0:1775767777.969832:0:36:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000100:00000001:2.0:1775767777.969832:0:9316:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775767777.969835:0:36:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000100:00080000:2.0:1775767777.969837:0:9316:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000001:1.0:1775767777.969838:0:36:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000100:00000040:3.0:1775767777.969839:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000020:00000001:1.0:1775767777.969840:0:36:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000001:1.0:1775767777.969843:0:36:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000400:00000010:3.0:1775767777.969845:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00020000:00000001:1.0:1775767777.969846:0:36:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00000100:00080000:2.0:1775767777.969846:0:9316:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00020000:00000001:1.0:1775767777.969849:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000001:2.0:1775767777.969850:0:9316:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000020:00000001:1.0:1775767777.969851:0:36:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000001:1.0:1775767777.969853:0:36:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000100:00000200:3.0:1775767777.969853:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294233600, offset 0 00000100:00080000:2.0:1775767777.969854:0:9316:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00020000:00000001:1.0:1775767777.969857:0:36:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775767777.969861:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767777.969862:0:9316:0:(import.c:484:import_select_connection()) Process entered 00020000:00000001:1.0:1775767777.969864:0:36:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000400:00000200:3.0:1775767777.969866:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000020:00000001:1.0:1775767777.969867:0:36:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000020:00000001:1.0:1775767777.969870:0:36:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775767777.969871:0:9316:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5030 00000020:00000001:1.0:1775767777.969872:0:36:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:1.0:1775767777.969875:0:36:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:1.0:1775767777.969877:0:36:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:1.0:1775767777.969878:0:36:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:1.0:1775767777.969880:0:36:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000400:00000200:3.0:1775767777.969881:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00001000:1.0:1775767777.969882:0:36:0:(cl_object.c:925:cl_env_put()) 1@ffff994206ababd0 00000020:00000010:1.0:1775767777.969887:0:36:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994208a8ec00. 00000020:00000010:1.0:1775767777.969892:0:36:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207cbaae0. 00000400:00000200:3.0:1775767777.969892:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000008:00000010:1.0:1775767777.969898:0:36:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207e00000. 00020000:00000010:1.0:1775767777.969902:0:36:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff9942031b30c0. 00000400:00000200:3.0:1775767777.969902:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00008000:00000010:1.0:1775767777.969907:0:36:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942054a8a98. 00000080:00000010:1.0:1775767777.969911:0:36:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99422030b0a8. 00000400:00000200:3.0:1775767777.969911:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000080:00000010:1.0:1775767777.969915:0:36:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff994205753d98. 00000400:00000200:3.0:1775767777.969918:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000020:00000010:1.0:1775767777.969919:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff99420120a200. 00000008:00000010:1.0:1775767777.969924:0:36:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff994210709080. 00000400:00000200:3.0:1775767777.969925:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00020000:00000010:1.0:1775767777.969927:0:36:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff9942057418c0. 00008000:00000010:1.0:1775767777.969931:0:36:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994219fec368. 00000080:00000010:1.0:1775767777.969935:0:36:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942057e24b0. 00000020:00000010:1.0:1775767777.969937:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994208a8f600. 00000020:00000010:1.0:1775767777.969940:0:36:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994206ababd0. 00020000:00000001:1.0:1775767777.969942:0:36:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00000400:00000200:3.0:1775767777.969942:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224998:224998:256:4294967295] 192.168.202.156@tcp LPNI seq info [224994:224994:8:4294967295] 00020000:00000001:1.0:1775767777.969944:0:36:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:1.0:1775767777.969947:0:36:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000080:00200000:1.0:1775767777.969950:0:36:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x10176:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000400:00000200:3.0:1775767777.969955:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000080:00000001:1.0:1775767777.969956:0:36:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:1.0:1775767777.969960:0:36:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:1.0:1775767777.969962:0:36:0:(cl_object.c:925:cl_env_put()) 1@ffff994206aba240 00000400:00000200:2.0:1775767777.969964:0:9316:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000010:1.0:1775767777.969966:0:36:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994201342e00. 00000400:00000200:2.0:1775767777.969972:0:9316:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000020:00000010:1.0:1775767777.969973:0:36:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207cba360. 00000100:00080000:2.0:1775767777.969976:0:9316:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000400:00000200:3.0:1775767777.969979:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000008:00000010:1.0:1775767777.969980:0:36:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207e02ca0. 00000100:00000040:2.0:1775767777.969983:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00020000:00000010:1.0:1775767777.969987:0:36:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff9942031b3480. 00000100:00000001:2.0:1775767777.969988:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00008000:00000010:1.0:1775767777.969993:0:36:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942054ab170. 00000100:00000040:2.0:1775767777.969994:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000800:00000200:3.0:1775767777.969997:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00000001:2.0:1775767777.969998:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000080:00000010:1.0:1775767777.970000:0:36:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99422030bf18. 00000020:00000001:2.0:1775767777.970003:0:9316:0:(genops.c:811:class_conn2export()) Process entered 00000080:00000010:1.0:1775767777.970005:0:36:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff994205751da8. 00000800:00000010:3.0:1775767777.970005:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000020:00000040:2.0:1775767777.970006:0:9316:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:2.0:1775767777.970010:0:9316:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775767777.970012:0:9316:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000010:1.0:1775767777.970013:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994201343200. 00000020:00000001:2.0:1775767777.970014:0:9316:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:2.0:1775767777.970019:0:9316:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000800:00000200:3.0:1775767777.970020:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000008:00000010:1.0:1775767777.970021:0:36:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff994210709ce0. 00000100:00000040:2.0:1775767777.970025:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00020000:00000010:1.0:1775767777.970026:0:36:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994205743180. 00000100:00000001:2.0:1775767777.970028:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:3.0:1775767777.970029:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00008000:00000010:1.0:1775767777.970032:0:36:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994219fec110. 00000080:00000010:1.0:1775767777.970037:0:36:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942057e2960. 00000100:00000040:2.0:1775767777.970050:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767777.970054:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000010:1.0:1775767777.970055:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994201343000. 00000800:00000200:3.1:1775767777.970058:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000040:2.0:1775767777.970059:0:9316:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000020:00000010:1.0:1775767777.970061:0:36:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994206aba240. 00000100:00080000:2.0:1775767777.970065:0:9316:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000800:00000200:3.1:1775767777.970067:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000080:00000001:1.0:1775767777.970068:0:36:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767777.970069:0:9316:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767777.970074:0:9316:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000080:00000001:1.0:1775767777.970076:0:36:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767777.970077:0:9316:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767777.970082:0:9316:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1775767777.970082:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000100:00000001:2.0:1775767777.970087:0:9316:0:(obd_class.h:890:obd_reconnect()) Process entered 00000800:00000010:3.0:1775767777.970087:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000008:00100000:2.0:1775767777.970089:0:9316:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775767777.970094:0:9316:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767777.970094:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:2.0:1775767777.970100:0:9316:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767777.970106:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000100:00000010:2.0:1775767777.970108:0:9316:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfed80. 00000020:00000040:2.0:1775767777.970112:0:9316:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000400:00000200:3.0:1775767777.970112:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767777.970117:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 02000000:00000001:2.0:1775767777.970118:0:9316:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775767777.970120:0:9316:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767777.970123:0:9316:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1775767777.970124:0:2399:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:2.0:1775767777.970125:0:9316:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775767777.970129:0:9316:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1775767777.970131:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfce00 x1862022294233600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767802 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1775767777.970134:0:9316:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1fc00. 00000100:00000001:2.0:1775767777.970140:0:9316:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767777.970151:0:9316:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfed80 x1862022294233856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767777.970178:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767777.970184:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfce00 x1862022294233600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767802 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767777.970207:0:9316:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775767777.970209:0:9316:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767777.970216:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767777.970217:0:9316:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb9b9eae] to pc [ptlrpcd_rcv+-1] req@ffff994207dfed80 x1862022294233856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767777.970221:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767777.970229:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767777.970231:0:9316:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775767777.970235:0:9316:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:3.0:1775767777.970237:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00080000:2.0:1775767777.970243:0:9316:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775767777.970243:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299704126 00000100:00000001:2.0:1775767777.970246:0:9316:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775767777.970249:0:9316:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000010:3.0:1775767777.970250:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942079810c0. 00000100:00000001:2.0:1775767777.970252:0:9316:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775767777.970257:0:9316:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5030 00000100:00000001:3.0:1775767777.970260:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000400:00000200:2.0:1775767777.970265:0:9316:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767777.970268:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299704126 00000400:00000200:2.0:1775767777.970271:0:9316:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775767777.970274:0:9316:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000010:3.0:1775767777.970274:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207981960. 00000100:00000040:2.0:1775767777.970280:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767777.970282:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1775767777.970282:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000040:2.0:1775767777.970286:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767777.970288:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:3.0:1775767777.970288:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1775767777.970289:0:9316:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775767777.970292:0:9316:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:2.0:1775767777.970295:0:9316:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:3.0:1775767777.970295:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775767777.970298:0:9316:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000100:00000001:3.0:1775767777.970300:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000020:00000001:2.0:1775767777.970302:0:9316:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:2.0:1775767777.970307:0:9316:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00100000:3.0:1775767777.970308:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767777/real 1775767777] req@ffff994207dfce00 x1862022294233600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775767777.970315:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767777.970321:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767777.970326:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000001:3.0:1775767777.970329:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1775767777.970330:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000001:3.0:1775767777.970332:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000040:2.0:1775767777.970335:0:9316:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000400:00000010:3.0:1775767777.970336:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000100:00080000:2.0:1775767777.970341:0:9316:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000400:00000200:3.0:1775767777.970342:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000100:00000001:2.0:1775767777.970346:0:9316:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1775767777.970346:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 02000000:00000001:2.0:1775767777.970352:0:9316:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1775767777.970352:0:2399:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:2.0:1775767777.970356:0:9316:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767777.970359:0:9316:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1775767777.970360:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfce00 x1862022294233600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767777.970361:0:9316:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775767777.970364:0:9316:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:2.0:1775767777.970370:0:9316:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767777.970375:0:9316:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1775767777.970380:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfce00 x1862022294233600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1775767777.970381:0:9316:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfe300. 00000020:00000040:2.0:1775767777.970387:0:9316:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:2.0:1775767777.970393:0:9316:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775767777.970396:0:9316:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767777.970397:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767777.970401:0:9316:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1775767777.970401:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1775767777.970404:0:9316:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1775767777.970406:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:2.0:1775767777.970407:0:9316:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767777.970409:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:2.0:1775767777.970414:0:9316:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1e400. 00000100:00100000:3.0:1775767777.970416:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfce00 x1862022294233600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767777.970419:0:9316:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767777.970429:0:9316:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfe300 x1862022294233984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767777.970430:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767777.970438:0:9316:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:3.0:1775767777.970440:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfce00 x1862022294233600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767802 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767777.970441:0:9316:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767777.970448:0:9316:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c661fef] to pc [ptlrpcd_rcv+-1] req@ffff994207dfe300 x1862022294233984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767777.970464:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767777.970471:0:9316:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767777.970471:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767777.970477:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:2.0:1775767777.970478:0:9316:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1775767777.970479:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767777.970482:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1775767777.970484:0:9316:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775767777.970485:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767777.970488:0:9316:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767777.970490:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767777.970491:0:9316:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775767777.970496:0:9316:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767777.970497:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00080000:2.0:1775767777.970502:0:9316:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5030 00000100:00000001:3.0:1775767777.970502:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000400:00000200:2.0:1775767777.970512:0:9316:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000040:3.0:1775767777.970514:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfce00 x1862022294233600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767802 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1775767777.970520:0:9316:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775767777.970524:0:9316:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775767777.970529:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767777.970532:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:3.0:1775767777.970534:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294233600:192.168.202.156@tcp:250:kworker.0 00000100:00000040:2.0:1775767777.970536:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767777.970541:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775767777.970544:0:9316:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775767777.970545:0:9316:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000100:00000001:3.0:1775767777.970545:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1775767777.970549:0:9316:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:3.0:1775767777.970550:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000020:00000040:2.0:1775767777.970551:0:9316:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:2.0:1775767777.970554:0:9316:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:2.0:1775767777.970558:0:9316:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:3.0:1775767777.970558:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfce00 x1862022294233600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767802 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775767777.970565:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767777.970568:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767777.970572:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767777.970576:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775767777.970580:0:9316:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00000001:3.0:1775767777.970582:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 00000100:00080000:2.0:1775767777.970583:0:9316:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775767777.970586:0:9316:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767777.970586:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1775767777.970590:0:9316:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000010:3.0:1775767777.970591:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b2800. 02000000:00000001:2.0:1775767777.970592:0:9316:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767777.970596:0:9316:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767777.970597:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1775767777.970598:0:9316:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775767777.970601:0:9316:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000020:00000001:3.0:1775767777.970601:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000008:00000001:2.0:1775767777.970605:0:9316:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775767777.970605:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000100:00000001:2.0:1775767777.970608:0:9316:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767777.970610:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000010:2.0:1775767777.970612:0:9316:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfc700. 02000000:00000010:3.0:1775767777.970614:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1f800. 00000020:00000040:2.0:1775767777.970615:0:9316:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:2.0:1775767777.970620:0:9316:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775767777.970621:0:9316:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767777.970621:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:2.0:1775767777.970625:0:9316:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767777.970625:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:2.0:1775767777.970628:0:9316:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000010:3.0:1775767777.970629:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfce00. 02000000:00000001:2.0:1775767777.970630:0:9316:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775767777.970635:0:9316:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1d000. 00000100:00000001:3.0:1775767777.970635:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767777.970638:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767777.970640:0:9316:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767777.970641:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767777.970645:0:9316:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfc700 x1862022294234112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767777.970649:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767777.970652:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767777.970654:0:9316:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767777.970656:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775767777.970657:0:9316:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767777.970660:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:2.0:1775767777.970663:0:9316:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a176dd1] to pc [ptlrpcd_rcv+-1] req@ffff994207dfc700 x1862022294234112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767777.970664:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767777.970672:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dff480 x1862022294233728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767777.970674:0:9316:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767777.970680:0:9316:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5040) 00000100:00000001:3.0:1775767777.970686:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767777.970688:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767777.970690:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767777.970693:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767777.970696:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767777.970700:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294233728:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767777.970703:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767777.970705:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767777.970707:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767777.970709:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767777.970712:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b2800. 02000000:00000001:3.0:1775767777.970714:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767777.970718:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767777.970721:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767777.970725:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767777.970728:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294233728, portal 10 00000100:00000001:3.0:1775767777.970730:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767777.970731:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611849856 : -112966097701760 : ffff994207dff480) 00000100:00000040:3.0:1775767777.970737:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dff480 x1862022294233728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767802 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767777.970744:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767777.970747:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767777.970749:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767777.970751:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294233728, offset 0 00000400:00000200:3.0:1775767777.970755:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767777.970760:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767777.970763:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767777.970766:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767777.970769:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767777.970772:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767777.970774:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767777.970777:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [224999:224999:256:4294967295] 192.168.202.156@tcp LPNI seq info [224995:224995:8:4294967295] 00000400:00000200:3.0:1775767777.970782:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767777.970788:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767777.970793:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767777.970797:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767777.970801:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767777.970804:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775767777.970809:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767777.970812:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000100:3.0:1775767777.970816:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767777.970817:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767777.970821:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767777.970824:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767777.970827:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767777.970829:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767777.970831:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767777.970835:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dff480 x1862022294233728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767802 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767777.970843:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767777.970846:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dff480 x1862022294233728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767802 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767777.970852:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767777.970853:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767777.970856:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767777.970860:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:3.0:1775767777.970863:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767777.970865:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299702803 00000100:00000001:3.0:1775767777.970867:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299704126 00000100:00000001:3.0:1775767777.970869:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299702803 00000100:00000001:3.0:1775767777.970871:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299704126 00000100:00000001:3.0:1775767777.970873:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767777.970875:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767777.970877:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767777.970880:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767777/real 1775767777] req@ffff994207dff480 x1862022294233728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767777.970894:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767777.970895:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767777.970897:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767777.970899:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767777.970900:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767777.970902:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767777.970905:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dff480 x1862022294233728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767777.970913:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dff480 x1862022294233728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767777.970919:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767777.970921:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767777.970923:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767777.970924:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767777.970928:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dff480 x1862022294233728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767777.970935:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767777.970939:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dff480 x1862022294233728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767802 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767777.970946:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767777.970948:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767777.970950:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767777.970951:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767777.970953:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767777.970956:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767777.970958:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767777.970962:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767777.970964:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767777.970969:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dff480 x1862022294233728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767802 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767777.970977:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294233728:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767777.970981:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767777.970985:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfed80 x1862022294233856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767777.970991:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767777.970993:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767777.970995:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767777.970997:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767777.970999:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767777.971002:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294233856:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767777.971006:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767777.971008:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767777.971010:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767777.971012:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767777.971015:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b0c00. 02000000:00000001:3.0:1775767777.971017:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767777.971021:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767777.971023:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767777.971026:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767777.971030:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294233856, portal 10 00000100:00000001:3.0:1775767777.971032:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767777.971033:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611848064 : -112966097703552 : ffff994207dfed80) 00000100:00000040:3.0:1775767777.971039:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfed80 x1862022294233856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767802 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767777.971062:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767777.971065:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767777.971068:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767777.971070:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294233856, offset 0 00000400:00000200:3.0:1775767777.971075:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767777.971079:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767777.971083:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767777.971086:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767777.971089:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767777.971092:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767777.971094:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767777.971097:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225000:225000:256:4294967295] 192.168.202.156@tcp LPNI seq info [224996:224996:8:4294967295] 00000400:00000200:3.0:1775767777.971102:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767777.971108:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767777.971113:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767777.971117:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767777.971121:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767777.971124:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775767777.971128:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767777.971131:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000100:3.0:1775767777.971135:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767777.971137:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767777.971140:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767777.971144:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767777.971147:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767777.971149:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767777.971151:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767777.971174:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfed80 x1862022294233856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767802 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767777.971183:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767777.971187:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfed80 x1862022294233856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767802 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767777.971193:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767777.971195:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767777.971199:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767777.971203:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:3.0:1775767777.971206:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767777.971209:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299702796 00000100:00000001:3.0:1775767777.971212:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299704126 00000100:00000001:3.0:1775767777.971214:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299702796 00000100:00000001:3.0:1775767777.971216:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299704126 00000100:00000001:3.0:1775767777.971219:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767777.971221:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767777.971223:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767777.971227:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767777/real 1775767777] req@ffff994207dfed80 x1862022294233856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767777.971235:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767777.971237:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767777.971238:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767777.971241:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767777.971243:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767777.971245:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767777.971248:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfed80 x1862022294233856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767777.971258:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfed80 x1862022294233856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767777.971265:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767777.971266:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767777.971268:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767777.971269:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767777.971272:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfed80 x1862022294233856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767777.971279:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767777.971283:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfed80 x1862022294233856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767802 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767777.971290:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767777.971293:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767777.971295:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767777.971296:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767777.971298:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767777.971301:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767777.971304:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767777.971308:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767777.971310:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767777.971314:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfed80 x1862022294233856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767802 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767777.971321:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294233856:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767777.971326:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767777.971329:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfe300 x1862022294233984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767777.971336:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767777.971337:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767777.971340:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767777.971342:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767777.971344:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767777.971347:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294233984:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767777.971351:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767777.971353:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767777.971354:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767777.971356:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767777.971360:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b2c00. 02000000:00000001:3.0:1775767777.971362:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767777.971366:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767777.971369:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767777.971372:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767777.971375:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294233984, portal 4 00000100:00000001:3.0:1775767777.971377:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767777.971378:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611845376 : -112966097706240 : ffff994207dfe300) 00000100:00000040:3.0:1775767777.971384:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfe300 x1862022294233984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767802 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767777.971390:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767777.971392:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767777.971395:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767777.971397:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294233984, offset 0 00000400:00000200:3.0:1775767777.971401:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767777.971405:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767777.971409:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767777.971412:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767777.971414:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767777.971417:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767777.971419:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767777.971423:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225001:225001:256:4294967295] 192.168.202.156@tcp LPNI seq info [224997:224997:8:4294967295] 00000400:00000200:3.0:1775767777.971427:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767777.971434:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767777.971439:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767777.971442:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767777.971446:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767777.971449:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775767777.971453:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767777.971455:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000100:3.0:1775767777.971460:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767777.971462:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767777.971464:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767777.971468:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767777.971471:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767777.971473:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767777.971475:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767777.971479:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfe300 x1862022294233984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767802 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767777.971493:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767777.971496:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfe300 x1862022294233984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767802 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767777.971502:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767777.971504:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767777.971506:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767777.971509:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:3.0:1775767777.971512:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767777.971515:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299702819 00000100:00000001:3.0:1775767777.971517:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299704126 00000100:00000001:3.0:1775767777.971520:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299702819 00000100:00000001:3.0:1775767777.971522:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299704126 00000100:00000001:3.0:1775767777.971524:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767777.971526:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767777.971528:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767777.971532:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767777/real 1775767777] req@ffff994207dfe300 x1862022294233984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767777.971540:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767777.971541:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767777.971543:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767777.971545:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767777.971547:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767777.971550:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767777.971553:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfe300 x1862022294233984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767777.971561:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfe300 x1862022294233984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767777.971568:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767777.971570:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767777.971572:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767777.971574:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767777.971577:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfe300 x1862022294233984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767777.971584:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767777.971588:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfe300 x1862022294233984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767802 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767777.971595:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767777.971598:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767777.971599:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767777.971601:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767777.971602:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767777.971606:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767777.971608:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767777.971612:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767777.971614:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767777.971618:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfe300 x1862022294233984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767802 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767777.971626:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294233984:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767777.971630:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767777.971632:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767777.971635:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dff480 x1862022294233728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767802 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767777.971642:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767777.971644:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767777.971646:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b2800. 02000000:00000001:3.0:1775767777.971648:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767777.971650:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767777.971652:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:3.0:1775767777.971654:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767777.971656:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1d400. 02000000:00000001:3.0:1775767777.971659:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767777.971660:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767777.971662:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dff480. 00000100:00000001:3.0:1775767777.971665:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767777.971666:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767777.971668:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767777.971671:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfed80 x1862022294233856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767802 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767777.971677:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767777.971679:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767777.971680:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b0c00. 02000000:00000001:3.0:1775767777.971683:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767777.971684:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767777.971685:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:3.0:1775767777.971687:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767777.971689:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1fc00. 02000000:00000001:3.0:1775767777.971691:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767777.971692:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767777.971693:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfed80. 00000100:00000001:3.0:1775767777.971696:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767777.971697:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767777.971699:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767777.971702:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfe300 x1862022294233984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767802 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767777.971708:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767777.971709:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767777.971711:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b2c00. 02000000:00000001:3.0:1775767777.971713:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767777.971714:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767777.971715:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:3.0:1775767777.971717:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767777.971719:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1e400. 02000000:00000001:3.0:1775767777.971722:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767777.971723:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767777.971725:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfe300. 00000100:00000001:3.0:1775767777.971727:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767777.971728:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767777.971730:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767777.971734:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767777.971735:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767777.971737:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767777.971738:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767777.971740:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767777.971743:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfc700 x1862022294234112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767777.971750:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767777.971752:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767777.971754:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767777.971756:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767777.971758:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767777.971761:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294234112:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767777.971764:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767777.971766:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767777.971767:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767777.971769:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767777.971772:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b2c00. 02000000:00000001:3.0:1775767777.971774:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767777.971777:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767777.971780:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767777.971782:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767777.971785:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294234112, portal 4 00000100:00000001:3.0:1775767777.971787:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767777.971789:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611838208 : -112966097713408 : ffff994207dfc700) 00000100:00000040:3.0:1775767777.971794:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfc700 x1862022294234112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767802 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767777.971800:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767777.971803:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767777.971805:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767777.971807:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294234112, offset 0 00000400:00000200:3.0:1775767777.971811:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767777.971816:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767777.971819:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767777.971822:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767777.971825:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767777.971827:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767777.971829:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767777.971833:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225002:225002:256:4294967295] 192.168.202.156@tcp LPNI seq info [224998:224998:8:4294967295] 00000400:00000200:3.0:1775767777.971837:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767777.971843:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767777.971847:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767777.971850:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767777.971854:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767777.971857:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775767777.971860:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767777.971863:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000100:3.0:1775767777.971867:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767777.971868:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767777.971871:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767777.971874:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767777.971877:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767777.971879:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767777.971881:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767777.971885:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfc700 x1862022294234112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767802 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767777.971892:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767777.971895:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfc700 x1862022294234112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767802 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767777.971902:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767777.971903:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767777.971906:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767777.971909:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:3.0:1775767777.971912:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767777.971915:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299702822 00000100:00000001:3.0:1775767777.971917:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299704126 00000100:00000001:3.0:1775767777.971919:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299702822 00000100:00000001:3.0:1775767777.971922:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299704126 00000100:00000001:3.0:1775767777.971924:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767777.971926:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767777.971928:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767777.971932:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767777/real 1775767777] req@ffff994207dfc700 x1862022294234112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767777.971940:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767777.971941:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767777.971943:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767777.971946:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767777.971947:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767777.971950:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767777.971952:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfc700 x1862022294234112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767777.971961:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfc700 x1862022294234112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767777.971968:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767777.971970:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767777.971972:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767777.971974:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767777.971977:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfc700 x1862022294234112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767777.971984:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767777.971988:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfc700 x1862022294234112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767802 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767777.971995:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767777.971998:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767777.971999:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767777.972001:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767777.972003:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767777.972007:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767777.972009:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767777.972013:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767777.972015:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767777.972019:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfc700 x1862022294234112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767802 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767777.972027:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294234112:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767777.972031:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767777.972033:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767777.972036:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfc700 x1862022294234112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767802 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767777.972057:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767777.972058:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767777.972060:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b2c00. 02000000:00000001:3.0:1775767777.972063:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767777.972064:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767777.972066:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:3.0:1775767777.972068:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767777.972069:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1d000. 02000000:00000001:3.0:1775767777.972072:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767777.972073:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767777.972075:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfc700. 00000100:00000001:3.0:1775767777.972077:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767777.972079:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767777.972081:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767777.972085:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767777.972086:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767777.972088:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767777.972090:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767778.993246:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767778.993253:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767778.993256:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767778.993262:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767778.993269:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767778.993275:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767778.993276:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767778.993280:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767778.993281:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767778.993283:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767778.993284:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767778.993289:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767778.993289:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767778.993295:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767778.993299:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767778.993304:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767778.993306:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767778.993308:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767779.057335:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767779.057339:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767779.057343:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775767779.313249:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299704126, at 4299704128 00000100:00000010:1.0:1775767779.313265:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207981960. 00000100:00000001:3.0:1775767780.017225:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767780.017227:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767780.017232:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767780.017236:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767780.017239:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767780.017242:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767780.017243:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:1.0:1775767780.017245:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767780.017250:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767780.017252:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767780.017256:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767780.017256:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767780.017267:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767780.017270:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767780.017272:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767780.017273:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767780.017276:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767780.017279:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767781.041266:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767781.041273:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767781.041278:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767781.041283:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767781.041289:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767781.041295:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767781.041300:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767781.041304:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767781.041307:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767781.041372:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767781.041374:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767781.041375:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767781.041377:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767781.041379:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767781.041380:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767781.041382:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767781.041384:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767781.041386:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767781.105300:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767781.105303:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767781.105306:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775767782.065242:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767782.065247:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767782.065250:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767782.065254:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767782.065260:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:3.0:1775767782.065261:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767782.065265:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767782.065267:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1775767782.065272:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767782.065272:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767782.065278:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767782.065282:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767782.065285:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767782.065289:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767782.065292:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767782.065294:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767782.065296:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767782.065299:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767783.089283:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767783.089288:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767783.089290:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767783.089294:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:3.0:1775767783.089295:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767783.089300:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767783.089301:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767783.089305:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767783.089306:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767783.089310:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767783.089312:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767783.089314:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767783.089315:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767783.089319:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767783.089319:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767783.089325:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767783.089328:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767783.089333:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767783.089350:0:9316:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767783.089355:0:9316:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775767783.089357:0:9316:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767783.089360:0:9316:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767783.089363:0:9316:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767783.089368:0:9316:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5035 00000400:00000200:3.0:1775767783.089377:0:9316:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767783.089381:0:9316:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767783.089384:0:9316:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767783.089388:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767783.089391:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767783.089393:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000080:00000001:2.0:1775767783.089395:0:36:0:(rw.c:1481:ll_writepages()) Process entered 00000100:00000001:3.0:1775767783.089395:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767783.089400:0:9316:0:(genops.c:811:class_conn2export()) Process entered 00000080:00000001:2.0:1775767783.089401:0:36:0:(file.c:5250:cl_sync_file_range()) Process entered 00000020:00000040:3.0:1775767783.089402:0:9316:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:2.0:1775767783.089404:0:36:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000001:3.0:1775767783.089405:0:9316:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767783.089408:0:9316:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:3.0:1775767783.089410:0:9316:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:3.0:1775767783.089412:0:9316:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000010:2.0:1775767783.089414:0:36:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994210b32630. 00000100:00000040:3.0:1775767783.089414:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767783.089419:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767783.089421:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000010:2.0:1775767783.089422:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994209d2cc00. 00000100:00000001:3.0:1775767783.089423:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767783.089426:0:9316:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000020:00000010:2.0:1775767783.089429:0:36:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994209d2dc00. 00000100:00080000:3.0:1775767783.089429:0:9316:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767783.089431:0:9316:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1775767783.089434:0:36:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942182d6120. 02000000:00000001:3.0:1775767783.089434:0:9316:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767783.089438:0:9316:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00000010:2.0:1775767783.089441:0:36:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207ccc2f0. 02000000:00000001:3.0:1775767783.089442:0:9316:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767783.089446:0:9316:0:(obd_class.h:890:obd_reconnect()) Process entered 00020000:00000010:2.0:1775767783.089447:0:36:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994220156e40. 00000100:00000001:3.0:1775767783.089450:0:9316:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:2.0:1775767783.089452:0:36:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942077a8e20. 00000100:00000010:3.0:1775767783.089458:0:9316:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8380. 00000080:00000010:2.0:1775767783.089459:0:36:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff9942087472a0. 00000080:00000010:2.0:1775767783.089464:0:36:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b8000. 00000020:00000040:3.0:1775767783.089464:0:9316:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 00000020:00000010:2.0:1775767783.089470:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994209d2d600. 02000000:00000001:3.0:1775767783.089472:0:9316:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000008:00000010:2.0:1775767783.089474:0:36:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942077cd600. 02000000:00000001:3.0:1775767783.089475:0:9316:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:2.0:1775767783.089479:0:36:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff99420876d680. 00000100:00000001:3.0:1775767783.089480:0:9316:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00008000:00000010:2.0:1775767783.089485:0:36:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076cf60. 02000000:00000001:3.0:1775767783.089485:0:9316:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000080:00000010:2.0:1775767783.089489:0:36:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff994205066870. 02000000:00000001:3.0:1775767783.089492:0:9316:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767783.089493:0:36:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107759908408 : -112965949643208 : ffff994210b32638) 00000020:00001000:2.0:1775767783.089497:0:36:0:(cl_object.c:831:cl_env_get()) 1@ffff994210b32630 00000020:00000001:2.0:1775767783.089500:0:36:0:(cl_object.c:151:cl_object_top()) ffff9942078a76c0 -> ffff9942078a76c0 00000020:00000001:2.0:1775767783.089502:0:36:0:(cl_io.c:125:__cl_io_init()) Process entered 02000000:00000010:3.0:1775767783.089503:0:9316:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2c00. 00000080:00000001:2.0:1775767783.089505:0:36:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:2.0:1775767783.089508:0:36:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:2.0:1775767783.089511:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775767783.089513:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00000001:3.0:1775767783.089513:0:9316:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775767783.089515:0:36:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775767783.089519:0:36:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017a:0x0]io ffff9942077b8060 type 6 ignore/verify layout 1/0 00020000:00000002:2.0:1775767783.089523:0:36:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207c1f9b0) owner 0000000000000000/ffff994201348000 00000100:00100000:3.0:1775767783.089524:0:9316:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8380 x1862022294234240/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00020000:00000001:2.0:1775767783.089528:0:36:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:2.0:1775767783.089531:0:36:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:2.0:1775767783.089535:0:36:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00020000:00000001:2.0:1775767783.089538:0:36:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767783.089539:0:9316:0:(jobid.c:936:lustre_get_jobid()) Process entered 00020000:00000001:2.0:1775767783.089541:0:36:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767783.089544:0:9316:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767783.089545:0:36:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:2.0:1775767783.089548:0:36:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767783.089552:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000100:00000040:3.0:1775767783.089553:0:9316:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055f5621a] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8380 x1862022294234240/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1775767783.089554:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:2.0:1775767783.089557:0:36:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00020000:00000002:2.0:1775767783.089559:0:36:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207c1f9b0) owner 0000000000000000/ffff994201348000 00000020:00000001:2.0:1775767783.089564:0:36:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767783.089569:0:36:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:2.0:1775767783.089571:0:36:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00020000:00000001:2.0:1775767783.089573:0:36:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00020000:00200000:2.0:1775767783.089575:0:36:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00000002:2.0:1775767783.089578:0:36:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:2.0:1775767783.089580:0:36:0:(lov_io.c:155:lov_sub_get()) Process entered 00020000:00000001:2.0:1775767783.089581:0:36:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000020:00000001:2.0:1775767783.089583:0:36:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00000001:3.0:1775767783.089583:0:9316:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1775767783.089585:0:36:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994210b32090. 00000100:00080000:3.0:1775767783.089587:0:9316:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000010:2.0:1775767783.089590:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994209d2de00. 00000100:00080000:3.0:1775767783.089591:0:9316:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000020:00000010:2.0:1775767783.089594:0:36:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994209d2d000. 00000100:00000001:3.0:1775767783.089594:0:9316:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000020:00000010:2.0:1775767783.089598:0:36:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942182d66c0. 00000100:00080000:3.0:1775767783.089598:0:9316:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767783.089603:0:9316:0:(import.c:484:import_select_connection()) Process entered 00000008:00000010:2.0:1775767783.089604:0:36:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207cc8b28. 00020000:00000010:2.0:1775767783.089609:0:36:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994220156840. 00000100:00080000:3.0:1775767783.089609:0:9316:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5035 00008000:00000010:2.0:1775767783.089616:0:36:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942077a9530. 00000400:00000200:3.0:1775767783.089619:0:9316:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767783.089624:0:9316:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767783.089626:0:9316:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000080:00000010:2.0:1775767783.089629:0:36:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994208747738. 00000100:00000040:3.0:1775767783.089633:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000080:00000010:2.0:1775767783.089634:0:36:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077ba910. 00000100:00000001:3.0:1775767783.089638:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000010:2.0:1775767783.089641:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9942060d4800. 00000100:00000040:3.0:1775767783.089643:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000008:00000010:2.0:1775767783.089646:0:36:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942077cd8c0. 00000100:00000001:3.0:1775767783.089648:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00020000:00000010:2.0:1775767783.089651:0:36:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff99420876e400. 00000020:00000001:3.0:1775767783.089652:0:9316:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767783.089655:0:9316:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00008000:00000010:2.0:1775767783.089656:0:36:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076c3f0. 00000020:00000001:3.0:1775767783.089659:0:9316:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000080:00000010:2.0:1775767783.089660:0:36:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942050663c0. 00000020:00000040:3.0:1775767783.089661:0:9316:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:2.0:1775767783.089665:0:36:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107759906968 : -112965949644648 : ffff994210b32098) 00000020:00000001:3.0:1775767783.089666:0:9316:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00001000:2.0:1775767783.089670:0:36:0:(cl_object.c:831:cl_env_get()) 1@ffff994210b32090 00000020:00000001:3.0:1775767783.089671:0:9316:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:2.0:1775767783.089675:0:36:0:(cl_object.c:151:cl_object_top()) ffff9942078a7788 -> ffff9942078a76c0 00000100:00000040:3.0:1775767783.089678:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000001:2.0:1775767783.089679:0:36:0:(cl_io.c:125:__cl_io_init()) Process entered 00000100:00000001:3.0:1775767783.089681:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:2.0:1775767783.089684:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775767783.089686:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00000040:3.0:1775767783.089686:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000001:2.0:1775767783.089689:0:36:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767783.089691:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00020000:00000001:2.0:1775767783.089693:0:36:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775767783.089695:0:9316:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00020000:00000001:2.0:1775767783.089697:0:36:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107621738216 : -112966087813400 : ffff99420876d6e8) 00000100:00080000:3.0:1775767783.089702:0:9316:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000020:00000001:2.0:1775767783.089703:0:36:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000100:00000001:3.0:1775767783.089705:0:9316:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767783.089706:0:36:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767783.089708:0:9316:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00020000:00000001:2.0:1775767783.089709:0:36:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767783.089713:0:36:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767783.089714:0:9316:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1775767783.089715:0:36:0:(cl_io.c:314:cl_io_lock()) Process entered 00020000:00000001:2.0:1775767783.089718:0:36:0:(lov_io.c:1108:lov_io_lock()) Process entered 02000000:00000001:3.0:1775767783.089719:0:9316:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767783.089721:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000001:3.0:1775767783.089722:0:9316:0:(obd_class.h:890:obd_reconnect()) Process entered 00000020:00000001:2.0:1775767783.089723:0:36:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:2.0:1775767783.089726:0:36:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:2.0:1775767783.089728:0:36:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1775767783.089728:0:9316:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000020:00000001:2.0:1775767783.089730:0:36:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1775767783.089734:0:9316:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767783.089735:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767783.089738:0:9316:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767783.089741:0:36:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767783.089743:0:36:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000100:00000010:3.0:1775767783.089744:0:9316:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8700. 00000020:00000001:2.0:1775767783.089745:0:36:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775767783.089749:0:9316:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:2.0:1775767783.089750:0:36:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767783.089754:0:36:0:(cl_io.c:528:cl_io_start()) Process entered 02000000:00000001:3.0:1775767783.089755:0:9316:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00020000:00000001:2.0:1775767783.089758:0:36:0:(lov_io.c:1114:lov_io_start()) Process entered 02000000:00000001:3.0:1775767783.089759:0:9316:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767783.089762:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000001:3.0:1775767783.089762:0:9316:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000020:00000001:2.0:1775767783.089765:0:36:0:(cl_io.c:528:cl_io_start()) Process entered 02000000:00000001:3.0:1775767783.089765:0:9316:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000008:00000001:2.0:1775767783.089768:0:36:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 02000000:00000001:3.0:1775767783.089768:0:9316:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767783.089773:0:9316:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0c00. 00000008:00000001:2.0:1775767783.089778:0:36:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000100:00000001:3.0:1775767783.089779:0:9316:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1775767783.089784:0:36:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:2.0:1775767783.089787:0:36:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000100:00100000:3.0:1775767783.089788:0:9316:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8700 x1862022294234368/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000008:00000001:2.0:1775767783.089789:0:36:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1775767783.089795:0:36:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000008:00000001:2.0:1775767783.089797:0:36:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000008:00000001:2.0:1775767783.089801:0:36:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107884691392 : -112965824860224 : ffff994218232fc0) 00000008:00000020:2.0:1775767783.089807:0:36:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994218232fc0 ready 0|-|- wr 1|-|+ rd 0|- 10 in flight 00000008:00000001:2.0:1775767783.089816:0:36:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000100:00000001:3.0:1775767783.089817:0:9316:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000008:00000020:2.0:1775767783.089818:0:36:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:2.0:1775767783.089822:0:36:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767783.089823:0:9316:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1775767783.089827:0:36:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000008:00000020:2.0:1775767783.089831:0:36:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994210675880 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000100:00000040:3.0:1775767783.089832:0:9316:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000faa25e3b] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8700 x1862022294234368/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:2.0:1775767783.089843:0:36:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1775767783.089849:0:36:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767783.089853:0:36:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767783.089858:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767783.089859:0:36:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767783.089860:0:9316:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767783.089862:0:36:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767783.089866:0:9316:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000001:2.0:1775767783.089867:0:36:0:(cl_io.c:555:cl_io_end()) Process entered 00020000:00000001:2.0:1775767783.089870:0:36:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00000100:00080000:3.0:1775767783.089871:0:9316:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00020000:00000001:2.0:1775767783.089872:0:36:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000100:00000001:3.0:1775767783.089874:0:9316:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000020:00000001:2.0:1775767783.089875:0:36:0:(cl_io.c:555:cl_io_end()) Process entered 00000100:00080000:3.0:1775767783.089878:0:9316:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000020:00000001:2.0:1775767783.089879:0:36:0:(cl_io.c:563:cl_io_end()) Process leaving 00020000:00000001:2.0:1775767783.089881:0:36:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767783.089884:0:9316:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767783.089889:0:9316:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5035 00020000:00000001:2.0:1775767783.089890:0:36:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000020:00000001:2.0:1775767783.089893:0:36:0:(cl_io.c:563:cl_io_end()) Process leaving 00000020:00000001:2.0:1775767783.089895:0:36:0:(cl_io.c:352:cl_io_unlock()) Process entered 00020000:00000001:2.0:1775767783.089899:0:36:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00000400:00000200:3.0:1775767783.089899:0:9316:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:2.0:1775767783.089902:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775767783.089904:0:36:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000020:00000001:2.0:1775767783.089905:0:36:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:2.0:1775767783.089908:0:36:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767783.089908:0:9316:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00020000:00000001:2.0:1775767783.089912:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767783.089916:0:36:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000100:00080000:3.0:1775767783.089919:0:9316:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000020:00000001:2.0:1775767783.089920:0:36:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:2.0:1775767783.089924:0:36:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000020:00000001:2.0:1775767783.089927:0:36:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000100:00000040:3.0:1775767783.089927:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000001:2.0:1775767783.089931:0:36:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000100:00000001:3.0:1775767783.089932:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00020000:00000001:2.0:1775767783.089934:0:36:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00020000:00000001:2.0:1775767783.089936:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775767783.089939:0:36:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000100:00000040:3.0:1775767783.089939:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000001:2.0:1775767783.089942:0:36:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000100:00000001:3.0:1775767783.089944:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00020000:00000001:2.0:1775767783.089945:0:36:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767783.089948:0:9316:0:(genops.c:811:class_conn2export()) Process entered 00020000:00000001:2.0:1775767783.089949:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775767783.089951:0:9316:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00020000:00000001:2.0:1775767783.089952:0:36:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000020:00000001:3.0:1775767783.089953:0:9316:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000001:2.0:1775767783.089954:0:36:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000020:00000040:3.0:1775767783.089956:0:9316:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:2.0:1775767783.089957:0:36:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767783.089960:0:9316:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:2.0:1775767783.089962:0:36:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:2.0:1775767783.089965:0:36:0:(lov_io.c:675:lov_io_fini()) Process entered 00000020:00000001:3.0:1775767783.089966:0:9316:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00020000:00000001:2.0:1775767783.089968:0:36:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:2.0:1775767783.089971:0:36:0:(cl_io.c:71:cl_io_fini()) Process entered 00000100:00000040:3.0:1775767783.089972:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000001:2.0:1775767783.089973:0:36:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000100:00000001:3.0:1775767783.089975:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00001000:2.0:1775767783.089976:0:36:0:(cl_object.c:925:cl_env_put()) 1@ffff994210b32090 00000100:00000040:3.0:1775767783.089980:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000010:2.0:1775767783.089981:0:36:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994209d2d000. 00000100:00000001:3.0:1775767783.089984:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767783.089989:0:9316:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000020:00000010:2.0:1775767783.090012:0:36:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942182d66c0. 00000100:00080000:3.0:1775767783.090014:0:9316:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000800:00000200:0.1:1775767783.090015:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000008:00000010:2.0:1775767783.090018:0:36:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207cc8b28. 00000100:00000001:3.0:1775767783.090021:0:9316:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775767783.090023:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00020000:00000010:2.0:1775767783.090024:0:36:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994220156840. 02000000:00000001:3.0:1775767783.090024:0:9316:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00008000:00000010:2.0:1775767783.090029:0:36:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942077a9530. 02000000:00000001:3.0:1775767783.090031:0:9316:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000010:2.0:1775767783.090033:0:36:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994208747738. 02000000:00000001:3.0:1775767783.090037:0:9316:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767783.090041:0:9316:0:(obd_class.h:890:obd_reconnect()) Process entered 00000080:00000010:2.0:1775767783.090045:0:36:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077ba910. 00000008:00100000:3.0:1775767783.090045:0:9316:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000020:00000010:2.0:1775767783.090052:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994209d2de00. 00000008:00000001:3.0:1775767783.090052:0:9316:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767783.090057:0:9316:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:2.0:1775767783.090059:0:36:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942077cd8c0. 00000100:00000010:3.0:1775767783.090065:0:9316:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a9180. 00020000:00000010:2.0:1775767783.090067:0:36:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff99420876e400. 00000020:00000040:3.0:1775767783.090071:0:9316:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00008000:00000010:2.0:1775767783.090073:0:36:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076c3f0. 02000000:00000001:3.0:1775767783.090079:0:9316:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000080:00000010:2.0:1775767783.090080:0:36:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942050663c0. 02000000:00000001:3.0:1775767783.090083:0:9316:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1775767783.090086:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9942060d4800. 00000100:00000001:3.0:1775767783.090089:0:9316:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000020:00000010:2.0:1775767783.090091:0:36:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994210b32090. 02000000:00000001:3.0:1775767783.090093:0:9316:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767783.090097:0:9316:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767783.090098:0:36:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:2.0:1775767783.090101:0:36:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:2.0:1775767783.090104:0:36:0:(vvp_io.c:280:vvp_io_fini()) Process entered 02000000:00000010:3.0:1775767783.090106:0:9316:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2800. 00000080:00200000:2.0:1775767783.090107:0:36:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00000001:3.0:1775767783.090112:0:9316:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775767783.090114:0:36:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:2.0:1775767783.090117:0:36:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:2.0:1775767783.090121:0:36:0:(cl_object.c:925:cl_env_put()) 1@ffff994210b32630 00000100:00100000:3.0:1775767783.090122:0:9316:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a9180 x1862022294234496/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000010:2.0:1775767783.090125:0:36:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994209d2dc00. 00000020:00000010:2.0:1775767783.090130:0:36:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942182d6120. 00000008:00000010:2.0:1775767783.090135:0:36:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207ccc2f0. 00020000:00000010:2.0:1775767783.090140:0:36:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994220156e40. 00000100:00000001:3.0:1775767783.090146:0:9316:0:(jobid.c:936:lustre_get_jobid()) Process entered 00008000:00000010:2.0:1775767783.090147:0:36:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942077a8e20. 00000100:00000001:3.0:1775767783.090151:0:9316:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:2.0:1775767783.090152:0:36:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff9942087472a0. 00000080:00000010:2.0:1775767783.090175:0:36:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b8000. 00000100:00000040:3.0:1775767783.090177:0:9316:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bdc2586] to pc [ptlrpcd_rcv+-1] req@ffff9942099a9180 x1862022294234496/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1775767783.090180:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994209d2cc00. 00000008:00000010:2.0:1775767783.090185:0:36:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942077cd600. 00000100:00000001:3.0:1775767783.090191:0:9316:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:2.0:1775767783.090192:0:36:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff99420876d680. 00008000:00000010:2.0:1775767783.090196:0:36:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076cf60. 00000100:00080000:3.0:1775767783.090196:0:9316:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000080:00000010:2.0:1775767783.090201:0:36:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff994205066870. 00000100:00080000:3.0:1775767783.090202:0:9316:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000020:00000010:2.0:1775767783.090206:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994209d2d600. 00000100:00000001:3.0:1775767783.090206:0:9316:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767783.090207:0:9316:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000020:00000010:2.0:1775767783.090209:0:36:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994210b32630. 00000100:00000001:3.0:1775767783.090211:0:9316:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767783.090215:0:9316:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5035 00000400:00000200:3.0:1775767783.090220:0:9316:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000080:00000001:2.0:1775767783.090221:0:36:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:2.0:1775767783.090226:0:36:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767783.090228:0:9316:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767783.090233:0:9316:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000080:00000001:2.0:1775767783.090234:0:36:0:(rw.c:1481:ll_writepages()) Process entered 00000100:00000040:3.0:1775767783.090237:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000080:00000001:2.0:1775767783.090238:0:36:0:(file.c:5250:cl_sync_file_range()) Process entered 00000020:00000001:2.0:1775767783.090241:0:36:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00000001:3.0:1775767783.090242:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000010:2.0:1775767783.090245:0:36:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994210b32630. 00000100:00000040:3.0:1775767783.090246:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000010:2.0:1775767783.090251:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9942060d4800. 00000100:00000001:3.0:1775767783.090251:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000010:2.0:1775767783.090257:0:36:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff9942060d5200. 00000020:00000001:3.0:1775767783.090257:0:9316:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767783.090260:0:9316:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000010:2.0:1775767783.090262:0:36:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942182d6120. 00000020:00000001:3.0:1775767783.090263:0:9316:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767783.090266:0:9316:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000008:00000010:2.0:1775767783.090267:0:36:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207ccc2f0. 00000020:00000001:3.0:1775767783.090269:0:9316:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00020000:00000010:2.0:1775767783.090270:0:36:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994220156e40. 00000020:00000001:3.0:1775767783.090274:0:9316:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00008000:00000010:2.0:1775767783.090275:0:36:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942077a8e20. 00000080:00000010:2.0:1775767783.090280:0:36:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff9942087472a0. 00000100:00000040:3.0:1775767783.090280:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000080:00000010:2.0:1775767783.090285:0:36:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b8000. 00000100:00000001:3.0:1775767783.090286:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000010:2.0:1775767783.090291:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9942060d4c00. 00000100:00000040:3.0:1775767783.090291:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000008:00000010:2.0:1775767783.090297:0:36:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942077cd600. 00000100:00000001:3.0:1775767783.090297:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00020000:00000010:2.0:1775767783.090302:0:36:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff99420876d680. 00000020:00000040:3.0:1775767783.090303:0:9316:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00008000:00000010:2.0:1775767783.090306:0:36:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076cf60. 00000100:00080000:3.0:1775767783.090308:0:9316:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000080:00000010:2.0:1775767783.090311:0:36:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff994205066870. 00000100:00000001:3.0:1775767783.090312:0:9316:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767783.090316:0:36:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107759908408 : -112965949643208 : ffff994210b32638) 02000000:00000001:3.0:1775767783.090316:0:9316:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767783.090320:0:9316:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00001000:2.0:1775767783.090321:0:36:0:(cl_object.c:831:cl_env_get()) 1@ffff994210b32630 02000000:00000001:3.0:1775767783.090323:0:9316:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767783.090325:0:36:0:(cl_object.c:151:cl_object_top()) ffff9942078a7f58 -> ffff9942078a7f58 00000100:00000001:3.0:1775767783.090327:0:9316:0:(obd_class.h:890:obd_reconnect()) Process entered 00000020:00000001:2.0:1775767783.090329:0:36:0:(cl_io.c:125:__cl_io_init()) Process entered 00000008:00100000:3.0:1775767783.090330:0:9316:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000080:00000001:2.0:1775767783.090333:0:36:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000008:00000001:3.0:1775767783.090334:0:9316:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000080:00200000:2.0:1775767783.090337:0:36:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017b:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000100:00000001:3.0:1775767783.090338:0:9316:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767783.090345:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000100:00000010:3.0:1775767783.090345:0:9316:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ab480. 00000020:00000001:2.0:1775767783.090349:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000040:3.0:1775767783.090350:0:9316:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00000080:00000001:2.0:1775767783.090352:0:36:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767783.090355:0:9316:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767783.090358:0:9316:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775767783.090359:0:36:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017b:0x0]io ffff9942077b8060 type 6 ignore/verify layout 1/0 00000100:00000001:3.0:1775767783.090362:0:9316:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767783.090365:0:9316:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00020000:00000002:2.0:1775767783.090368:0:36:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207c1f000) owner 0000000000000000/ffff994201348000 02000000:00000001:3.0:1775767783.090368:0:9316:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767783.090375:0:36:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 02000000:00000010:3.0:1775767783.090375:0:9316:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0000. 00020000:00000001:2.0:1775767783.090379:0:36:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00000100:00000001:3.0:1775767783.090380:0:9316:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767783.090382:0:36:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00020000:00000001:2.0:1775767783.090386:0:36:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767783.090389:0:9316:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ab480 x1862022294234624/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00020000:00000001:2.0:1775767783.090392:0:36:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:2.0:1775767783.090399:0:36:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:2.0:1775767783.090402:0:36:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767783.090408:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000100:00000001:3.0:1775767783.090408:0:9316:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000020:00000001:2.0:1775767783.090411:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00000001:3.0:1775767783.090413:0:9316:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767783.090414:0:36:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00020000:00000002:2.0:1775767783.090417:0:36:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207c1f000) owner 0000000000000000/ffff994201348000 00000100:00000040:3.0:1775767783.090419:0:9316:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000067a0ba74] to pc [ptlrpcd_rcv+-1] req@ffff9942099ab480 x1862022294234624/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1775767783.090421:0:36:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767783.090425:0:36:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:2.0:1775767783.090428:0:36:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00020000:00000001:2.0:1775767783.090430:0:36:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00020000:00200000:2.0:1775767783.090433:0:36:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00000002:2.0:1775767783.090437:0:36:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00000100:00000001:3.0:1775767783.090439:0:9316:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767783.090441:0:36:0:(lov_io.c:155:lov_sub_get()) Process entered 00020000:00000001:2.0:1775767783.090444:0:36:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000100:00080000:3.0:1775767783.090445:0:9316:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000001:2.0:1775767783.090447:0:36:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775767783.090451:0:36:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994210b32090. 00000100:00080000:3.0:1775767783.090455:0:9316:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000020:00000010:2.0:1775767783.090457:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9942060d5800. 00000100:00000001:3.0:1775767783.090460:0:9316:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000020:00000010:2.0:1775767783.090462:0:36:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff9942060d5a00. 00000100:00080000:3.0:1775767783.090463:0:9316:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000020:00000010:2.0:1775767783.090465:0:36:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942182d66c0. 00000008:00000010:2.0:1775767783.090471:0:36:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207cc8b28. 00000100:00000001:3.0:1775767783.090471:0:9316:0:(import.c:484:import_select_connection()) Process entered 00020000:00000010:2.0:1775767783.090476:0:36:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994220156840. 00000100:00080000:3.0:1775767783.090478:0:9316:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5035 00008000:00000010:2.0:1775767783.090481:0:36:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942077a9530. 00000080:00000010:2.0:1775767783.090486:0:36:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994208747738. 00000080:00000010:2.0:1775767783.090491:0:36:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077ba910. 00000400:00000200:3.0:1775767783.090491:0:9316:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000010:2.0:1775767783.090497:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994209d2d600. 00000400:00000200:3.0:1775767783.090500:0:9316:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000008:00000010:2.0:1775767783.090501:0:36:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942077cd8c0. 00000100:00080000:3.0:1775767783.090506:0:9316:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00020000:00000010:2.0:1775767783.090507:0:36:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff99420876e400. 00008000:00000010:2.0:1775767783.090512:0:36:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076c3f0. 00000100:00000040:3.0:1775767783.090513:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000080:00000010:2.0:1775767783.090517:0:36:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942050663c0. 00000100:00000001:3.0:1775767783.090518:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:2.0:1775767783.090522:0:36:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107759906968 : -112965949644648 : ffff994210b32098) 00000100:00000040:3.0:1775767783.090523:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00001000:2.0:1775767783.090527:0:36:0:(cl_object.c:831:cl_env_get()) 1@ffff994210b32090 00000100:00000001:3.0:1775767783.090528:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775767783.090530:0:36:0:(cl_object.c:151:cl_object_top()) ffff9942078a75f8 -> ffff9942078a7f58 00000020:00000001:3.0:1775767783.090533:0:9316:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000001:2.0:1775767783.090535:0:36:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000040:3.0:1775767783.090537:0:9316:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:2.0:1775767783.090538:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:3.0:1775767783.090539:0:9316:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000001:2.0:1775767783.090540:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000040:3.0:1775767783.090542:0:9316:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:2.0:1775767783.090544:0:36:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767783.090546:0:9316:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00020000:00000001:2.0:1775767783.090547:0:36:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767783.090551:0:36:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107621738216 : -112966087813400 : ffff99420876d6e8) 00000020:00000001:3.0:1775767783.090551:0:9316:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:2.0:1775767783.090557:0:36:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000100:00000040:3.0:1775767783.090559:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000001:2.0:1775767783.090561:0:36:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767783.090566:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00020000:00000001:2.0:1775767783.090567:0:36:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767783.090572:0:36:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767783.090572:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000001:2.0:1775767783.090578:0:36:0:(cl_io.c:314:cl_io_lock()) Process entered 00000100:00000001:3.0:1775767783.090580:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00020000:00000001:2.0:1775767783.090583:0:36:0:(lov_io.c:1108:lov_io_lock()) Process entered 00000020:00000040:3.0:1775767783.090587:0:9316:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00020000:00000001:2.0:1775767783.090588:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775767783.090593:0:36:0:(cl_io.c:314:cl_io_lock()) Process entered 00000100:00080000:3.0:1775767783.090595:0:9316:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000020:00000001:2.0:1775767783.090597:0:36:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:2.0:1775767783.090600:0:36:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767783.090602:0:9316:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767783.090605:0:36:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767783.090607:0:9316:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00020000:00000001:2.0:1775767783.090610:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767783.090612:0:9316:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:2.0:1775767783.090617:0:36:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767783.090618:0:9316:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767783.090622:0:36:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000100:00000001:3.0:1775767783.090623:0:9316:0:(obd_class.h:890:obd_reconnect()) Process entered 00000020:00000001:2.0:1775767783.090626:0:36:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1775767783.090627:0:9316:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000020:00000001:2.0:1775767783.090633:0:36:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1775767783.090634:0:9316:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767783.090638:0:36:0:(cl_io.c:528:cl_io_start()) Process entered 00000100:00000001:3.0:1775767783.090641:0:9316:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767783.090642:0:36:0:(lov_io.c:1114:lov_io_start()) Process entered 00020000:00000001:2.0:1775767783.090645:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775767783.090647:0:36:0:(cl_io.c:528:cl_io_start()) Process entered 00000100:00000010:3.0:1775767783.090649:0:9316:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8e00. 00000008:00000001:2.0:1775767783.090651:0:36:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000008:00000001:2.0:1775767783.090654:0:36:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000020:00000040:3.0:1775767783.090654:0:9316:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00000008:00000001:2.0:1775767783.090660:0:36:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 02000000:00000001:3.0:1775767783.090661:0:9316:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000008:00000020:2.0:1775767783.090662:0:36:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:2.0:1775767783.090665:0:36:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1775767783.090665:0:9316:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1775767783.090671:0:36:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000100:00000001:3.0:1775767783.090671:0:9316:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000008:00000001:2.0:1775767783.090675:0:36:0:(osc_cache.c:2287:osc_next_obj()) Process entered 02000000:00000001:3.0:1775767783.090675:0:9316:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000008:00000001:2.0:1775767783.090679:0:36:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107884688256 : -112965824863360 : ffff994218232380) 02000000:00000001:3.0:1775767783.090680:0:9316:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000020:2.0:1775767783.090684:0:36:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994218232380 ready 0|-|- wr 1|-|+ rd 0|- 9 in flight 00000008:00000001:2.0:1775767783.090687:0:36:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 02000000:00000010:3.0:1775767783.090689:0:9316:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0400. 00000008:00000020:2.0:1775767783.090690:0:36:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:2.0:1775767783.090694:0:36:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767783.090696:0:9316:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1775767783.090698:0:36:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000008:00000020:2.0:1775767783.090702:0:36:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994210674fc0 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000100:00100000:3.0:1775767783.090705:0:9316:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8e00 x1862022294234752/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000008:00000001:2.0:1775767783.090710:0:36:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1775767783.090715:0:36:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767783.090720:0:36:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767783.090725:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767783.090730:0:36:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767783.090734:0:36:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767783.090740:0:36:0:(cl_io.c:555:cl_io_end()) Process entered 00020000:00000001:2.0:1775767783.090743:0:36:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00020000:00000001:2.0:1775767783.090747:0:36:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000100:00000001:3.0:1775767783.090749:0:9316:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000020:00000001:2.0:1775767783.090750:0:36:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000001:2.0:1775767783.090753:0:36:0:(cl_io.c:563:cl_io_end()) Process leaving 00000100:00000001:3.0:1775767783.090754:0:9316:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767783.090755:0:36:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767783.090759:0:36:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000020:00000001:2.0:1775767783.090761:0:36:0:(cl_io.c:563:cl_io_end()) Process leaving 00000020:00000001:2.0:1775767783.090764:0:36:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000100:00000040:3.0:1775767783.090765:0:9316:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000864a6866] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8e00 x1862022294234752/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:2.0:1775767783.090766:0:36:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00020000:00000001:2.0:1775767783.090768:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775767783.090770:0:36:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000020:00000001:2.0:1775767783.090772:0:36:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:2.0:1775767783.090774:0:36:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767783.090778:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767783.090782:0:36:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000020:00000001:2.0:1775767783.090784:0:36:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:2.0:1775767783.090787:0:36:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000020:00000001:2.0:1775767783.090789:0:36:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000001:2.0:1775767783.090793:0:36:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:2.0:1775767783.090795:0:36:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00020000:00000001:2.0:1775767783.090798:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775767783.090801:0:36:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000100:00000001:3.0:1775767783.090801:0:9316:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767783.090803:0:36:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00020000:00000001:2.0:1775767783.090807:0:36:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767783.090809:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767783.090809:0:9316:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5045) 00020000:00000001:2.0:1775767783.090813:0:36:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000020:00000001:2.0:1775767783.090816:0:36:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000020:00000001:2.0:1775767783.090818:0:36:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767783.090820:0:36:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:2.0:1775767783.090822:0:36:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:2.0:1775767783.090824:0:36:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:2.0:1775767783.090826:0:36:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:2.0:1775767783.090828:0:36:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:2.0:1775767783.090830:0:36:0:(cl_object.c:925:cl_env_put()) 1@ffff994210b32090 00000020:00000010:2.0:1775767783.090833:0:36:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff9942060d5a00. 00000020:00000010:2.0:1775767783.090837:0:36:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942182d66c0. 00000008:00000010:2.0:1775767783.090840:0:36:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207cc8b28. 00020000:00000010:2.0:1775767783.090846:0:36:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994220156840. 00008000:00000010:2.0:1775767783.090849:0:36:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942077a9530. 00000080:00000010:2.0:1775767783.090852:0:36:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994208747738. 00000080:00000010:2.0:1775767783.090857:0:36:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077ba910. 00000020:00000010:2.0:1775767783.090862:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9942060d5800. 00000008:00000010:2.0:1775767783.090864:0:36:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942077cd8c0. 00020000:00000010:2.0:1775767783.090868:0:36:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff99420876e400. 00008000:00000010:2.0:1775767783.090872:0:36:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076c3f0. 00000080:00000010:2.0:1775767783.090874:0:36:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942050663c0. 00000020:00000010:2.0:1775767783.090879:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994209d2d600. 00000020:00000010:2.0:1775767783.090881:0:36:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994210b32090. 00020000:00000001:2.0:1775767783.090884:0:36:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:2.0:1775767783.090886:0:36:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:2.0:1775767783.090888:0:36:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000080:00200000:2.0:1775767783.090890:0:36:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:2.0:1775767783.090893:0:36:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:2.0:1775767783.090895:0:36:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:2.0:1775767783.090897:0:36:0:(cl_object.c:925:cl_env_put()) 1@ffff994210b32630 00000020:00000010:2.0:1775767783.090901:0:36:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff9942060d5200. 00000020:00000010:2.0:1775767783.090903:0:36:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942182d6120. 00000008:00000010:2.0:1775767783.090907:0:36:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207ccc2f0. 00020000:00000010:2.0:1775767783.090910:0:36:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994220156e40. 00008000:00000010:2.0:1775767783.090914:0:36:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942077a8e20. 00000080:00000010:2.0:1775767783.090918:0:36:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff9942087472a0. 00000080:00000010:2.0:1775767783.090921:0:36:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b8000. 00000020:00000010:2.0:1775767783.090925:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9942060d4800. 00000008:00000010:2.0:1775767783.090928:0:36:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942077cd600. 00020000:00000010:2.0:1775767783.090933:0:36:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff99420876d680. 00008000:00000010:2.0:1775767783.090938:0:36:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076cf60. 00000080:00000010:2.0:1775767783.090939:0:36:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff994205066870. 00000020:00000010:2.0:1775767783.090945:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9942060d4c00. 00000020:00000010:2.0:1775767783.090947:0:36:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994210b32630. 00000080:00000001:2.0:1775767783.090951:0:36:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:2.0:1775767783.090953:0:36:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775767783.090957:0:36:0:(rw.c:1481:ll_writepages()) Process entered 00000080:00000001:2.0:1775767783.090959:0:36:0:(file.c:5250:cl_sync_file_range()) Process entered 00000020:00000001:2.0:1775767783.090961:0:36:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775767783.090964:0:36:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994210b32630. 00000020:00000010:2.0:1775767783.090968:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994209d2d600. 00000020:00000010:2.0:1775767783.090971:0:36:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994209d2cc00. 00000020:00000010:2.0:1775767783.090974:0:36:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942182d6120. 00000008:00000010:2.0:1775767783.090978:0:36:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207ccc2f0. 00020000:00000010:2.0:1775767783.090981:0:36:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994220156e40. 00008000:00000010:2.0:1775767783.090983:0:36:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942077a8e20. 00000080:00000010:2.0:1775767783.090987:0:36:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff9942087472a0. 00000080:00000010:2.0:1775767783.090990:0:36:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b8000. 00000020:00000010:2.0:1775767783.091009:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994209d2dc00. 00000008:00000010:2.0:1775767783.091013:0:36:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942077cd600. 00020000:00000010:2.0:1775767783.091016:0:36:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff99420876d680. 00008000:00000010:2.0:1775767783.091020:0:36:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076cf60. 00000080:00000010:2.0:1775767783.091023:0:36:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff994205066870. 00000020:00000001:2.0:1775767783.091027:0:36:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107759908408 : -112965949643208 : ffff994210b32638) 00000020:00001000:2.0:1775767783.091030:0:36:0:(cl_object.c:831:cl_env_get()) 1@ffff994210b32630 00000020:00000001:2.0:1775767783.091033:0:36:0:(cl_object.c:151:cl_object_top()) ffff99420791a850 -> ffff99420791a850 00000020:00000001:2.0:1775767783.091036:0:36:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:2.0:1775767783.091038:0:36:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:2.0:1775767783.091041:0:36:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017f:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:2.0:1775767783.091045:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775767783.091048:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:2.0:1775767783.091051:0:36:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775767783.091056:0:36:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017f:0x0]io ffff9942077b8060 type 6 ignore/verify layout 1/0 00020000:00000002:2.0:1775767783.091062:0:36:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207dfbd90) owner 0000000000000000/ffff994201348000 00020000:00000001:2.0:1775767783.091068:0:36:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:2.0:1775767783.091071:0:36:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:2.0:1775767783.091074:0:36:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00020000:00000001:2.0:1775767783.091077:0:36:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767783.091082:0:36:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:0.1:1775767783.091082:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00020000:00000001:2.0:1775767783.091087:0:36:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00000800:00000200:0.1:1775767783.091088:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00020000:00000001:2.0:1775767783.091090:0:36:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767783.091095:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775767783.091098:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:2.0:1775767783.091100:0:36:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00020000:00000002:2.0:1775767783.091104:0:36:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207dfbd90) owner 0000000000000000/ffff994201348000 00000020:00000001:2.0:1775767783.091109:0:36:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767783.091114:0:36:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:2.0:1775767783.091118:0:36:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00020000:00000001:2.0:1775767783.091119:0:36:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00020000:00200000:2.0:1775767783.091123:0:36:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00000002:2.0:1775767783.091125:0:36:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:2.0:1775767783.091130:0:36:0:(lov_io.c:155:lov_sub_get()) Process entered 00020000:00000001:2.0:1775767783.091130:0:36:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000020:00000001:2.0:1775767783.091133:0:36:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775767783.091136:0:36:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994210b32090. 00000020:00000010:2.0:1775767783.091140:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994209d2de00. 00000020:00000010:2.0:1775767783.091143:0:36:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994209d2d000. 00000020:00000010:2.0:1775767783.091147:0:36:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942182d66c0. 00000008:00000010:2.0:1775767783.091151:0:36:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207cc8b28. 00020000:00000010:2.0:1775767783.091171:0:36:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994220156840. 00008000:00000010:2.0:1775767783.091173:0:36:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942077a9530. 00000080:00000010:2.0:1775767783.091177:0:36:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994208747738. 00000080:00000010:2.0:1775767783.091180:0:36:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077ba910. 00000020:00000010:2.0:1775767783.091184:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9942060d4c00. 00000008:00000010:2.0:1775767783.091187:0:36:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942077cd8c0. 00020000:00000010:2.0:1775767783.091191:0:36:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff99420876e400. 00008000:00000010:2.0:1775767783.091193:0:36:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076c3f0. 00000080:00000010:2.0:1775767783.091197:0:36:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942050663c0. 00000020:00000001:2.0:1775767783.091200:0:36:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107759906968 : -112965949644648 : ffff994210b32098) 00000020:00001000:2.0:1775767783.091204:0:36:0:(cl_object.c:831:cl_env_get()) 1@ffff994210b32090 00000020:00000001:2.0:1775767783.091207:0:36:0:(cl_object.c:151:cl_object_top()) ffff99420791a210 -> ffff99420791a850 00000020:00000001:2.0:1775767783.091210:0:36:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:2.0:1775767783.091213:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775767783.091214:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:2.0:1775767783.091218:0:36:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767783.091222:0:36:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767783.091223:0:36:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107621738216 : -112966087813400 : ffff99420876d6e8) 00000020:00000001:2.0:1775767783.091225:0:36:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000020:00000001:2.0:1775767783.091227:0:36:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767783.091230:0:36:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767783.091231:0:36:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767783.091233:0:36:0:(cl_io.c:314:cl_io_lock()) Process entered 00020000:00000001:2.0:1775767783.091234:0:36:0:(lov_io.c:1108:lov_io_lock()) Process entered 00020000:00000001:2.0:1775767783.091237:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775767783.091239:0:36:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:2.0:1775767783.091241:0:36:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:2.0:1775767783.091244:0:36:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767783.091246:0:36:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767783.091249:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767783.091252:0:36:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767783.091254:0:36:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:2.0:1775767783.091255:0:36:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767783.091257:0:36:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767783.091260:0:36:0:(cl_io.c:528:cl_io_start()) Process entered 00020000:00000001:2.0:1775767783.091261:0:36:0:(lov_io.c:1114:lov_io_start()) Process entered 00020000:00000001:2.0:1775767783.091263:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775767783.091266:0:36:0:(cl_io.c:528:cl_io_start()) Process entered 00000008:00000001:2.0:1775767783.091267:0:36:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000008:00000001:2.0:1775767783.091269:0:36:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000008:00000001:2.0:1775767783.091272:0:36:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:2.0:1775767783.091274:0:36:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:2.0:1775767783.091276:0:36:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1775767783.091278:0:36:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000008:00000001:2.0:1775767783.091280:0:36:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000008:00000001:2.0:1775767783.091283:0:36:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107754938304 : -112965954613312 : ffff994210674fc0) 00000008:00000020:2.0:1775767783.091287:0:36:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994210674fc0 ready 0|-|- wr 1|-|+ rd 0|- 9 in flight 00000008:00000001:2.0:1775767783.091291:0:36:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:2.0:1775767783.091292:0:36:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:2.0:1775767783.091294:0:36:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1775767783.091298:0:36:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000008:00000020:2.0:1775767783.091300:0:36:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994218232380 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000008:00000001:2.0:1775767783.091307:0:36:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1775767783.091311:0:36:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767783.091314:0:36:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767783.091316:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767783.091319:0:36:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767783.091322:0:36:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767783.091324:0:36:0:(cl_io.c:555:cl_io_end()) Process entered 00020000:00000001:2.0:1775767783.091326:0:36:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00020000:00000001:2.0:1775767783.091328:0:36:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000020:00000001:2.0:1775767783.091329:0:36:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000001:2.0:1775767783.091331:0:36:0:(cl_io.c:563:cl_io_end()) Process leaving 00020000:00000001:2.0:1775767783.091334:0:36:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767783.091337:0:36:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000020:00000001:2.0:1775767783.091339:0:36:0:(cl_io.c:563:cl_io_end()) Process leaving 00000020:00000001:2.0:1775767783.091341:0:36:0:(cl_io.c:352:cl_io_unlock()) Process entered 00020000:00000001:2.0:1775767783.091343:0:36:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00020000:00000001:2.0:1775767783.091344:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775767783.091346:0:36:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000020:00000001:2.0:1775767783.091347:0:36:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:2.0:1775767783.091349:0:36:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767783.091352:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767783.091354:0:36:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000020:00000001:2.0:1775767783.091356:0:36:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:2.0:1775767783.091357:0:36:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000020:00000001:2.0:1775767783.091359:0:36:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000001:2.0:1775767783.091361:0:36:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:2.0:1775767783.091363:0:36:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00020000:00000001:2.0:1775767783.091365:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775767783.091366:0:36:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000001:2.0:1775767783.091369:0:36:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00020000:00000001:2.0:1775767783.091370:0:36:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767783.091373:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767783.091375:0:36:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000020:00000001:2.0:1775767783.091376:0:36:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000020:00000001:2.0:1775767783.091378:0:36:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767783.091380:0:36:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:2.0:1775767783.091382:0:36:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:2.0:1775767783.091383:0:36:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:2.0:1775767783.091385:0:36:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:2.0:1775767783.091387:0:36:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:2.0:1775767783.091389:0:36:0:(cl_object.c:925:cl_env_put()) 1@ffff994210b32090 00000020:00000010:2.0:1775767783.091391:0:36:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994209d2d000. 00000020:00000010:2.0:1775767783.091396:0:36:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942182d66c0. 00000008:00000010:2.0:1775767783.091399:0:36:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207cc8b28. 00020000:00000010:2.0:1775767783.091402:0:36:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994220156840. 00008000:00000010:2.0:1775767783.091405:0:36:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942077a9530. 00000080:00000010:2.0:1775767783.091410:0:36:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994208747738. 00000080:00000010:2.0:1775767783.091413:0:36:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077ba910. 00000020:00000010:2.0:1775767783.091416:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994209d2de00. 00000008:00000010:2.0:1775767783.091420:0:36:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942077cd8c0. 00020000:00000010:2.0:1775767783.091423:0:36:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff99420876e400. 00008000:00000010:2.0:1775767783.091427:0:36:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076c3f0. 00000080:00000010:2.0:1775767783.091431:0:36:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942050663c0. 00000020:00000010:2.0:1775767783.091435:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9942060d4c00. 00000020:00000010:2.0:1775767783.091438:0:36:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994210b32090. 00020000:00000001:2.0:1775767783.091441:0:36:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:2.0:1775767783.091442:0:36:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:2.0:1775767783.091444:0:36:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000080:00200000:2.0:1775767783.091446:0:36:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017f:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:2.0:1775767783.091453:0:36:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:2.0:1775767783.091454:0:36:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:2.0:1775767783.091456:0:36:0:(cl_object.c:925:cl_env_put()) 1@ffff994210b32630 00000020:00000010:2.0:1775767783.091459:0:36:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994209d2cc00. 00000020:00000010:2.0:1775767783.091463:0:36:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942182d6120. 00000008:00000010:2.0:1775767783.091466:0:36:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207ccc2f0. 00020000:00000010:2.0:1775767783.091470:0:36:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994220156e40. 00008000:00000010:2.0:1775767783.091473:0:36:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942077a8e20. 00000080:00000010:2.0:1775767783.091476:0:36:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff9942087472a0. 00000080:00000010:2.0:1775767783.091479:0:36:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b8000. 00000020:00000010:2.0:1775767783.091482:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994209d2d600. 00000008:00000010:2.0:1775767783.091486:0:36:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942077cd600. 00020000:00000010:2.0:1775767783.091489:0:36:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff99420876d680. 00008000:00000010:2.0:1775767783.091492:0:36:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076cf60. 00000080:00000010:2.0:1775767783.091496:0:36:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff994205066870. 00000020:00000010:2.0:1775767783.091500:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994209d2dc00. 00000020:00000010:2.0:1775767783.091504:0:36:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994210b32630. 00000080:00000001:2.0:1775767783.091509:0:36:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:2.0:1775767783.091514:0:36:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775767783.091619:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767783.091620:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775767783.091800:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767783.091802:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775767783.092078:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767783.092080:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000020:00000001:2.0:1775767783.153368:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767783.153373:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767783.153376:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775767784.113255:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767784.113262:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767784.113264:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767784.113269:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767784.113272:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767784.113277:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767784.113280:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767784.113282:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767784.113286:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767784.113286:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767784.113300:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767784.113309:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767784.113322:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767784.113330:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767784.113341:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767784.113355:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767784.113357:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767784.113360:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767784.433258:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299709246, at 4299709248 00000100:00000010:1.0:1775767784.433274:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3d80. 00000100:00000001:1.0:1775767785.137253:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767785.137253:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767785.137260:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767785.137260:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767785.137263:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767785.137264:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767785.137270:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775767785.137271:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767785.137277:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767785.137278:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767785.137281:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767785.137282:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767785.137284:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767785.137288:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767785.137291:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767785.137295:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767785.137297:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767785.137299:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767785.201368:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767785.201371:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767785.201373:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775767786.161223:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767786.161224:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767786.161229:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767786.161230:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767786.161231:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767786.161232:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767786.161240:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775767786.161240:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767786.161246:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767786.161249:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767786.161250:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767786.161251:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767786.161252:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767786.161255:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767786.161256:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767786.161256:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767786.161260:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767786.161262:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767786.161263:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00100000:3.0:1775767786.161264:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775767786.161266:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767786.161269:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767786.161269:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767786.161273:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767786.161276:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767786.161277:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767786.161279:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767787.185247:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767787.185248:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767787.185254:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767787.185255:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767787.185257:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767787.185257:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767787.185258:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767787.185263:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:1.0:1775767787.185264:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775767787.185265:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775767787.185266:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767787.185271:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767787.185272:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767787.185272:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767787.185277:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767787.185277:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767787.185281:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767787.185281:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767787.185284:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767787.185285:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767787.185290:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767787.185292:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767787.185294:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767787.185297:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767787.185297:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767787.185301:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767787.185304:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767787.249350:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767787.249354:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767787.249358:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775767788.209238:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767788.209247:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767788.209247:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767788.209250:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767788.209251:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767788.209252:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767788.209255:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767788.209257:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767788.209257:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767788.209263:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:1.0:1775767788.209265:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775767788.209267:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767788.209268:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767788.209273:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767788.209273:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767788.209273:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775767788.209279:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767788.209279:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767788.209284:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767788.209285:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767788.209290:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767788.209291:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767788.209292:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767788.209294:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767788.209296:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767788.209300:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767788.209305:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767788.209504:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767788.209509:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767788.209514:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767788.209524:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fe300 x1862022294234880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767788.209541:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767788.209545:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767788.209551:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767788.209555:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767788.209561:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767788.209569:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294234880:192.168.202.156@tcp:250:kworker.0 00000100:00000001:3.0:1775767788.209578:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767788.209584:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767788.209588:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767788.209592:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767788.209600:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b3400. 02000000:00000001:3.0:1775767788.209604:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767788.209615:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767788.209621:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767788.209630:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767788.209639:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294234880, portal 25 00000100:00000001:3.0:1775767788.209644:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767788.209648:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641205504 : -112966068346112 : ffff9942099fe300) 00000100:00000040:3.0:1775767788.209659:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fe300 x1862022294234880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767813 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767788.209675:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767788.209681:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767788.209685:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767788.209691:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294234880, offset 0 00000400:00000200:3.0:1775767788.209702:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767788.209714:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767788.209723:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767788.209732:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767788.209739:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767788.209745:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767788.209752:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767788.209761:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225008:225008:256:4294967295] 192.168.202.156@tcp LPNI seq info [225004:225004:8:4294967295] 00000400:00000200:3.0:1775767788.209768:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767788.209786:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767788.209800:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767788.209809:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767788.209821:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767788.209828:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775767788.209841:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767788.209849:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000100:3.0:1775767788.209860:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767788.209865:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767788.209874:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767788.209887:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767788.209895:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767788.209900:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767788.209908:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767788.209918:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fe300 x1862022294234880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767813 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767788.209964:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767788.209973:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fe300 x1862022294234880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767813 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767788.209988:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767788.209993:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767788.210002:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767788.210011:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767788.210015:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299714365 00000100:00000010:3.0:1775767788.210024:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207981720. 00000100:00000001:3.0:1775767788.210034:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1775767788.210043:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299714365 00000100:00000010:3.0:1775767788.210047:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942079815a0. 00000100:00000001:3.0:1775767788.210051:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1775767788.210056:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767788.210061:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767788.210065:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767788.210072:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767788/real 1775767788] req@ffff9942099fe300 x1862022294234880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767788.210089:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767788.210092:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767788.210095:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767788.210100:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767788.210103:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767788.210106:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767788.210112:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fe300 x1862022294234880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767788.210128:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fe300 x1862022294234880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767788.210139:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767788.210140:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767788.210144:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767788.210145:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767788.210153:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fe300 x1862022294234880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767788.210176:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767788.210180:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fe300 x1862022294234880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767813 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767788.210185:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767788.210188:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767788.210189:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767788.210191:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767788.210192:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767788.210194:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767788.210197:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767788.210205:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767788.210211:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767788.210218:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fe300 x1862022294234880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767813 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767788.210235:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294234880:192.168.202.156@tcp:250:kworker.0 00000100:00000001:3.0:1775767788.210243:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767788.210245:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767788.210247:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fe300 x1862022294234880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767813 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767788.210252:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767788.210254:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767788.210257:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b3400. 02000000:00000001:3.0:1775767788.210260:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767788.210261:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767788.210263:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:3.0:1775767788.210264:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767788.210266:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002bc00. 02000000:00000001:3.0:1775767788.210271:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767788.210272:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767788.210274:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fe300. 00000100:00000001:3.0:1775767788.210277:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767788.210278:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767788.210280:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767788.210285:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767788.210287:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767788.210288:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767788.210290:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767788.210291:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767788.210293:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fd880 x1862022294235008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767788.210298:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767788.210299:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767788.210301:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767788.210302:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767788.210304:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767788.210305:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294235008:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767788.210308:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767788.210309:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767788.210310:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767788.210311:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767788.210313:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b3400. 02000000:00000001:3.0:1775767788.210314:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767788.210317:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767788.210319:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767788.210322:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767788.210324:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294235008, portal 10 00000100:00000001:3.0:1775767788.210325:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767788.210326:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641202816 : -112966068348800 : ffff9942099fd880) 00000100:00000040:3.0:1775767788.210331:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fd880 x1862022294235008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767813 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767788.210335:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767788.210337:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767788.210338:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767788.210340:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294235008, offset 0 00000400:00000200:3.0:1775767788.210342:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767788.210345:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767788.210348:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767788.210350:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767788.210352:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767788.210354:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767788.210355:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767788.210358:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225009:225009:256:4294967295] 192.168.202.156@tcp LPNI seq info [225005:225005:8:4294967295] 00000400:00000200:3.0:1775767788.210360:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767788.210365:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767788.210369:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767788.210371:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767788.210374:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767788.210376:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775767788.210379:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767788.210381:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000100:3.0:1775767788.210384:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767788.210385:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767788.210387:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767788.210389:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767788.210391:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767788.210393:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767788.210394:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767788.210396:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fd880 x1862022294235008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767813 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767788.210402:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767788.210403:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fd880 x1862022294235008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767813 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767788.210407:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767788.210408:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767788.210410:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767788.210413:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:3.0:1775767788.210415:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767788.210416:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299713043 00000100:00000001:3.0:1775767788.210418:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299714365 00000100:00000001:3.0:1775767788.210419:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299713043 00000100:00000001:3.0:1775767788.210421:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299714365 00000100:00000001:3.0:1775767788.210422:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767788.210423:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767788.210425:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767788.210427:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767788/real 1775767788] req@ffff9942099fd880 x1862022294235008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767788.210432:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767788.210433:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767788.210434:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767788.210436:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767788.210437:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767788.210438:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767788.210439:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fd880 x1862022294235008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767788.210444:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fd880 x1862022294235008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767788.210448:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767788.210449:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767788.210450:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767788.210451:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767788.210453:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fd880 x1862022294235008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767788.210458:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767788.210460:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fd880 x1862022294235008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767813 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767788.210464:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767788.210465:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767788.210466:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767788.210467:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767788.210468:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767788.210470:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767788.210471:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767788.210473:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767788.210475:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767788.210477:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fd880 x1862022294235008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767813 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767788.210483:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294235008:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767788.210485:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767788.210487:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ff480 x1862022294235136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767788.210491:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767788.210492:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767788.210493:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767788.210494:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767788.210495:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767788.210497:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294235136:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767788.210498:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767788.210499:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767788.210500:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767788.210502:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767788.210503:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b2000. 02000000:00000001:3.0:1775767788.210505:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767788.210507:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767788.210508:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767788.210510:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767788.210512:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294235136, portal 10 00000100:00000001:3.0:1775767788.210513:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767788.210514:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641209984 : -112966068341632 : ffff9942099ff480) 00000100:00000040:3.0:1775767788.210517:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ff480 x1862022294235136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767813 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767788.210521:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767788.210523:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767788.210524:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767788.210525:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294235136, offset 0 00000400:00000200:3.0:1775767788.210528:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767788.210530:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767788.210532:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767788.210534:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767788.210535:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767788.210537:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767788.210539:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767788.210541:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225010:225010:256:4294967295] 192.168.202.156@tcp LPNI seq info [225006:225006:8:4294967295] 00000400:00000200:3.0:1775767788.210543:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767788.210547:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767788.210549:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767788.210551:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767788.210553:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767788.210555:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775767788.210558:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767788.210560:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000100:3.0:1775767788.210564:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767788.210566:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767788.210569:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767788.210572:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767788.210575:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767788.210577:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767788.210579:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767788.210582:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ff480 x1862022294235136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767813 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767788.210590:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767788.210593:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ff480 x1862022294235136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767813 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767788.210600:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767788.210602:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767788.210605:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767788.210608:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:3.0:1775767788.210612:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767788.210615:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299713035 00000100:00000001:3.0:1775767788.210617:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299714365 00000100:00000001:3.0:1775767788.210620:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299713035 00000100:00000001:3.0:1775767788.210622:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299714365 00000100:00000001:3.0:1775767788.210624:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767788.210627:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767788.210628:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767788.210632:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767788/real 1775767788] req@ffff9942099ff480 x1862022294235136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767788.210639:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767788.210640:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767788.210642:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767788.210643:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767788.210644:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767788.210646:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767788.210648:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ff480 x1862022294235136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767788.210654:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ff480 x1862022294235136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767788.210658:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767788.210659:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767788.210660:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767788.210661:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767788.210663:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ff480 x1862022294235136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767788.210667:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767788.210669:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ff480 x1862022294235136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767813 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767788.210673:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767788.210675:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767788.210676:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767788.210677:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767788.210678:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767788.210680:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767788.210682:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767788.210684:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767788.210685:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767788.210687:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ff480 x1862022294235136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767813 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767788.210692:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294235136:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767788.210695:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767788.210697:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fd180 x1862022294235264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767788.210701:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767788.210702:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767788.210703:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767788.210704:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767788.210706:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767788.210707:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294235264:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767788.210709:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767788.210710:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767788.210712:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767788.210713:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767788.210715:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b1000. 02000000:00000001:3.0:1775767788.210717:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767788.210719:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767788.210721:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767788.210723:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767788.210725:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294235264, portal 4 00000100:00000001:3.0:1775767788.210726:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767788.210727:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641201024 : -112966068350592 : ffff9942099fd180) 00000100:00000040:3.0:1775767788.210731:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fd180 x1862022294235264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767813 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767788.210736:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767788.210737:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767788.210739:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767788.210740:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294235264, offset 0 00000400:00000200:3.0:1775767788.210743:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767788.210745:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767788.210748:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767788.210749:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767788.210751:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767788.210753:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767788.210754:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767788.210757:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225011:225011:256:4294967295] 192.168.202.156@tcp LPNI seq info [225007:225007:8:4294967295] 00000400:00000200:3.0:1775767788.210759:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767788.210764:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767788.210767:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767788.210769:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767788.210771:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767788.210773:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775767788.210776:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767788.210778:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000100:3.0:1775767788.210780:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767788.210782:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767788.210784:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767788.210786:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767788.210788:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767788.210789:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767788.210790:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767788.210792:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fd180 x1862022294235264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767813 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767788.210802:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767788.210804:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fd180 x1862022294235264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767813 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767788.210808:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767788.210809:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767788.210811:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767788.210813:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:3.0:1775767788.210815:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767788.210816:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299713058 00000100:00000001:3.0:1775767788.210817:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299714365 00000100:00000001:3.0:1775767788.210819:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299713058 00000100:00000001:3.0:1775767788.210820:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299714365 00000100:00000001:3.0:1775767788.210822:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767788.210823:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767788.210824:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767788.210826:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767788/real 1775767788] req@ffff9942099fd180 x1862022294235264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767788.210831:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767788.210831:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767788.210832:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767788.210833:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767788.210834:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767788.210835:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767788.210837:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fd180 x1862022294235264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767788.210842:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fd180 x1862022294235264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767788.210846:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767788.210848:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767788.210849:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767788.210850:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767788.210852:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fd180 x1862022294235264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767788.210856:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767788.210858:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fd180 x1862022294235264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767813 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767788.210862:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767788.210863:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767788.210864:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767788.210865:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767788.210866:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767788.210868:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767788.210870:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767788.210872:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767788.210873:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767788.210875:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fd180 x1862022294235264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767813 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767788.210879:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294235264:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767788.210882:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767788.210883:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fed80 x1862022294235392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767788.210888:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767788.210889:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767788.210890:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767788.210891:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767788.210892:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767788.210894:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294235392:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767788.210896:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767788.210897:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767788.210899:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767788.210900:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767788.210903:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b0400. 02000000:00000001:3.0:1775767788.210904:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767788.210905:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767788.210907:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767788.210909:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767788.210910:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294235392, portal 4 00000100:00000001:3.0:1775767788.210911:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767788.210912:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641208192 : -112966068343424 : ffff9942099fed80) 00000100:00000040:3.0:1775767788.210916:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fed80 x1862022294235392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767813 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767788.210920:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767788.210921:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767788.210923:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767788.210924:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294235392, offset 0 00000400:00000200:3.0:1775767788.210926:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767788.210928:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767788.210930:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767788.210932:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767788.210933:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767788.210934:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767788.210936:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767788.210938:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225012:225012:256:4294967295] 192.168.202.156@tcp LPNI seq info [225008:225008:8:4294967295] 00000400:00000200:3.0:1775767788.210941:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767788.210963:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767788.210968:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767788.210972:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767788.210976:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767788.210979:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775767788.210983:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767788.210985:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000100:3.0:1775767788.210989:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767788.210991:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767788.210994:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767788.210997:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767788.211000:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767788.211002:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767788.211004:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767788.211008:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fed80 x1862022294235392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767813 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767788.211016:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767788.211019:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fed80 x1862022294235392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767813 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767788.211025:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767788.211028:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767788.211031:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767788.211034:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:3.0:1775767788.211037:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767788.211039:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299713061 00000100:00000001:3.0:1775767788.211041:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299714365 00000100:00000001:3.0:1775767788.211044:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299713061 00000100:00000001:3.0:1775767788.211046:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299714365 00000100:00000001:3.0:1775767788.211049:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767788.211051:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767788.211053:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767788.211056:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767788/real 1775767788] req@ffff9942099fed80 x1862022294235392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767788.211064:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767788.211065:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767788.211066:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767788.211068:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767788.211069:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767788.211071:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767788.211073:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fed80 x1862022294235392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767788.211078:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fed80 x1862022294235392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767788.211082:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767788.211083:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767788.211084:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767788.211085:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767788.211087:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fed80 x1862022294235392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767788.211091:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767788.211093:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fed80 x1862022294235392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767813 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767788.211098:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767788.211099:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767788.211100:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767788.211101:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767788.211102:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767788.211106:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767788.211108:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767788.211111:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767788.211113:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767788.211117:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fed80 x1862022294235392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767813 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767788.211131:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294235392:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767788.211134:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767788.211136:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767788.211138:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fd880 x1862022294235008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767813 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767788.211144:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767788.211146:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767788.211148:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b3400. 02000000:00000001:3.0:1775767788.211150:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767788.211151:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767788.211153:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:3.0:1775767788.211174:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767788.211176:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028400. 02000000:00000001:3.0:1775767788.211179:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767788.211181:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767788.211182:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fd880. 00000100:00000001:3.0:1775767788.211185:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767788.211186:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767788.211188:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767788.211191:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ff480 x1862022294235136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767813 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767788.211198:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767788.211199:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767788.211200:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b2000. 02000000:00000001:3.0:1775767788.211202:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767788.211204:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767788.211205:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:3.0:1775767788.211206:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767788.211208:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028000. 02000000:00000001:3.0:1775767788.211209:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767788.211210:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767788.211212:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ff480. 00000100:00000001:3.0:1775767788.211214:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767788.211215:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767788.211217:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767788.211220:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fd180 x1862022294235264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767813 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767788.211227:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767788.211228:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767788.211229:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b1000. 02000000:00000001:3.0:1775767788.211231:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767788.211232:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767788.211233:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:3.0:1775767788.211235:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767788.211236:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029400. 02000000:00000001:3.0:1775767788.211239:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767788.211240:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767788.211241:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fd180. 00000100:00000001:3.0:1775767788.211243:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767788.211244:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767788.211246:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767788.211249:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fed80 x1862022294235392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767813 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767788.211255:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767788.211256:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767788.211258:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b0400. 02000000:00000001:3.0:1775767788.211260:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767788.211261:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767788.211262:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:3.0:1775767788.211264:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767788.211265:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002a800. 02000000:00000001:3.0:1775767788.211267:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767788.211268:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767788.211269:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fed80. 00000100:00000001:3.0:1775767788.211271:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767788.211272:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767788.211274:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767788.211278:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767788.211279:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767788.211281:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767788.211283:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767789.233258:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767789.233258:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767789.233261:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767789.233265:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767789.233265:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767789.233267:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767789.233269:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767789.233269:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767789.233270:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767789.233277:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775767789.233277:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:1.0:1775767789.233278:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775767789.233283:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767789.233283:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767789.233284:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767789.233287:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767789.233288:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767789.233288:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767789.233294:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767789.233296:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767789.233298:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767789.233301:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767789.233304:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767789.233306:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767789.233308:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767789.233311:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767789.233315:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767789.297318:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767789.297322:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767789.297325:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775767789.553216:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299714365, at 4299714368 00000100:00000010:1.0:1775767789.553229:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942079815a0. 00000100:00000001:3.0:1775767790.257317:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767790.257319:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767790.257319:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767790.257331:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767790.257331:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767790.257332:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767790.257335:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767790.257337:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767790.257337:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767790.257345:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775767790.257346:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:2.0:1775767790.257347:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767790.257355:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767790.257356:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767790.257357:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767790.257367:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767790.257367:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767790.257371:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767790.257374:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767790.257376:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767790.257379:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767790.257384:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767790.257386:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767790.257388:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767790.257390:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767790.257393:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767790.257397:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767791.281266:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767791.281272:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767791.281274:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767791.281280:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767791.281283:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767791.281289:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767791.281293:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767791.281297:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767791.281297:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767791.281306:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767791.281306:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767791.281315:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767791.281328:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767791.281335:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767791.281340:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767791.281349:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767791.281352:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767791.281357:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767791.345379:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767791.345384:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767791.345388:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775767792.305296:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767792.305302:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767792.305307:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767792.305308:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767792.305312:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767792.305314:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767792.305327:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:1.0:1775767792.305328:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775767792.305341:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767792.305342:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767792.305353:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767792.305354:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767792.305365:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767792.305368:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767792.305371:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767792.305379:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767792.305384:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767792.305389:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767793.329247:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767793.329250:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767793.329255:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767793.329255:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767793.329257:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767793.329259:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767793.329269:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775767793.329269:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767793.329275:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767793.329276:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767793.329279:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775767793.329280:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775767793.329281:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767793.329284:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767793.329287:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775767793.329288:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775767793.329294:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767793.329294:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775767793.329295:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:3.0:1775767793.329298:0:36:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775767793.329299:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767793.329303:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775767793.329304:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767793.329310:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5045 00000400:00000200:3.0:1775767793.329321:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767793.329327:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767793.329329:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767793.329332:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767793.329335:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767793.329338:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767793.329340:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767793.329343:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767793.329345:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:3.0:1775767793.329347:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767793.329349:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:3.0:1775767793.329352:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:3.0:1775767793.329354:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:3.0:1775767793.329356:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767793.329357:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767793.329359:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767793.329361:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767793.329363:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:3.0:1775767793.329366:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767793.329367:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767793.329370:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767793.329372:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767793.329374:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767793.329376:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775767793.329378:0:36:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767793.329386:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aad80. 00000020:00000040:3.0:1775767793.329388:0:36:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:3.0:1775767793.329391:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767793.329392:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767793.329394:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767793.329396:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767793.329400:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767793.329406:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0400. 00000100:00000001:3.0:1775767793.329411:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767793.329418:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aad80 x1862022294235520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767793.329426:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767793.329430:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767793.329433:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000045ccf08] to pc [ptlrpcd_rcv+-1] req@ffff9942099aad80 x1862022294235520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767793.329455:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767793.329458:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767793.329460:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775767793.329461:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767793.329462:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767793.329464:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767793.329466:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5045 00000400:00000200:3.0:1775767793.329468:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767793.329470:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767793.329472:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767793.329475:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767793.329477:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767793.329479:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767793.329480:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767793.329481:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767793.329483:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:3.0:1775767793.329484:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767793.329486:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:3.0:1775767793.329487:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:3.0:1775767793.329488:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:3.0:1775767793.329490:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767793.329492:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767793.329496:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767793.329498:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767793.329501:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00080000:3.0:1775767793.329504:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767793.329506:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767793.329509:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767793.329514:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767793.329519:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767793.329523:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767793.329528:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775767793.329532:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767793.329535:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767793.329540:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aa680. 00000020:00000040:3.0:1775767793.329542:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:3.0:1775767793.329545:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767793.329546:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767793.329550:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767793.329553:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767793.329554:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767793.329567:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b1000. 00000100:00000001:3.0:1775767793.329571:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767793.329577:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aa680 x1862022294235648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767793.329589:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767793.329590:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767793.329593:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006cb1597d] to pc [ptlrpcd_rcv+-1] req@ffff9942099aa680 x1862022294235648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767793.329605:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767793.329608:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767793.329611:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775767793.329615:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767793.329616:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767793.329618:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767793.329621:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5045 00000400:00000200:3.0:1775767793.329625:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767793.329628:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767793.329631:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767793.329636:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767793.329639:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767793.329643:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767793.329645:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767793.329650:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767793.329652:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:3.0:1775767793.329654:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767793.329657:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:3.0:1775767793.329662:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:3.0:1775767793.329664:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:3.0:1775767793.329671:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767793.329673:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767793.329678:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767793.329682:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767793.329685:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:3.0:1775767793.329689:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767793.329694:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767793.329695:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767793.329697:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767793.329702:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767793.329705:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767793.329708:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775767793.329711:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767793.329714:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767793.329718:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ab800. 00000020:00000040:3.0:1775767793.329721:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:3.0:1775767793.329724:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767793.329727:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767793.329730:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767793.329733:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767793.329736:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767793.329740:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2000. 00000100:00000001:3.0:1775767793.329744:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767793.329751:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ab800 x1862022294235776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767793.329767:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767793.329770:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767793.329777:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000089df80c0] to pc [ptlrpcd_rcv+-1] req@ffff9942099ab800 x1862022294235776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767793.329786:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767793.329788:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767793.329793:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775767793.329796:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767793.329797:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767793.329799:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767793.329803:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5045 00000800:00000200:0.1:1775767793.329807:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000400:00000200:3.0:1775767793.329808:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767793.329813:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000800:00000200:0.1:1775767793.329813:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000100:00080000:3.0:1775767793.329818:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767793.329823:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767793.329827:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767793.329829:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767793.329831:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767793.329832:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767793.329833:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:3.0:1775767793.329834:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767793.329835:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:3.0:1775767793.329838:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:3.0:1775767793.329840:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:3.0:1775767793.329842:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767793.329844:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767793.329849:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767793.329852:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767793.329855:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:3.0:1775767793.329858:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767793.329861:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767793.329863:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767793.329865:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767793.329868:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767793.329871:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767793.329873:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:3.0:1775767793.329878:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767793.329882:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767793.329886:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aa300. 00000020:00000040:3.0:1775767793.329888:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:3.0:1775767793.329891:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767793.329906:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767793.329907:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767793.329908:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767793.329909:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767793.329912:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b3400. 00000100:00000001:3.0:1775767793.329915:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767793.329923:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aa300 x1862022294235904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767793.329937:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767793.329939:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767793.329943:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f33bdf15] to pc [ptlrpcd_rcv+-1] req@ffff9942099aa300 x1862022294235904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767793.329957:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767793.329959:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767793.329962:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775767793.329965:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767793.329966:0:36:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767793.329968:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767793.329971:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5045 00000400:00000200:3.0:1775767793.329975:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767793.329978:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767793.329982:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767793.329986:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767793.329989:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767793.329991:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767793.329995:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767793.329997:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767793.329999:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:3.0:1775767793.330003:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767793.330005:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:3.0:1775767793.330009:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:3.0:1775767793.330011:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:3.0:1775767793.330016:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767793.330017:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767793.330020:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767793.330022:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767793.330025:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:3.0:1775767793.330029:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767793.330033:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767793.330037:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767793.330040:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767793.330045:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767793.330047:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767793.330054:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:3.0:1775767793.330058:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767793.330060:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767793.330063:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8a80. 00000020:00000040:3.0:1775767793.330064:0:36:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:3.0:1775767793.330066:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767793.330067:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767793.330068:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767793.330070:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767793.330072:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767793.330075:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0000. 00000100:00000001:3.0:1775767793.330079:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767793.330083:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8a80 x1862022294236032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767793.330093:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767793.330097:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767793.330107:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006c3265af] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8a80 x1862022294236032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767793.330129:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767793.330136:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5055) 00000800:00000200:0.1:1775767793.330351:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767793.330352:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775767793.330540:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767793.330542:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775767793.330857:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767793.330859:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775767793.331212:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767793.331215:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000020:00000001:2.0:1775767793.393348:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767793.393351:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767793.393353:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775767794.353457:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767794.353459:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767794.353460:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767794.353462:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767794.353464:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767794.353465:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767794.353467:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767794.353468:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767794.353469:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767794.353549:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767794.353552:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767794.353553:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767794.353557:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767794.353559:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767794.353569:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767794.353571:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767794.353572:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767794.353573:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767794.673243:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299719485, at 4299719488 00000100:00000010:1.0:1775767794.673253:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3f00. 00000100:00000001:2.0:1775767795.377246:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767795.377249:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767795.377252:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767795.377254:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767795.377255:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767795.377258:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767795.377261:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:1.0:1775767795.377262:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775767795.377264:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767795.377267:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767795.377269:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767795.377270:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767795.377275:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767795.377279:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767795.377281:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767795.377281:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767795.377285:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767795.377288:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767795.441335:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767795.441339:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767795.441343:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775767796.401276:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767796.401276:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767796.401282:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767796.401283:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767796.401284:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767796.401286:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767796.401287:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767796.401292:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:1.0:1775767796.401294:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775767796.401294:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767796.401294:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767796.401300:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767796.401302:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767796.401302:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767796.401303:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767796.401308:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767796.401308:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767796.401314:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767796.401314:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767796.401318:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767796.401323:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767796.401323:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767796.401326:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767796.401328:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767796.401330:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767796.401334:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767796.401337:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767797.425234:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767797.425235:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767797.425241:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767797.425242:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767797.425243:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767797.425243:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767797.425246:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767797.425247:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:1.0:1775767797.425249:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775767797.425250:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767797.425251:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767797.425253:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:2.0:1775767797.425254:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767797.425257:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767797.425258:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767797.425259:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767797.425261:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767797.425263:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767797.425263:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767797.425266:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767797.425269:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767797.425274:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767797.425277:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767797.425281:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767797.425281:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767797.425284:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767797.425287:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767797.489366:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767797.489369:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767797.489372:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775767798.449253:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:3.0:1775767798.449253:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775767798.449258:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767798.449261:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767798.449265:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00080000:3.0:1775767798.449266:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775767798.449271:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767798.449272:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775767798.449279:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775767798.449279:0:36:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775767798.449285:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767798.449288:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767798.449291:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767798.449292:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767798.449303:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5050 00000400:00000200:3.0:1775767798.449316:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767798.449318:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767798.449324:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767798.449325:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767798.449327:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00100000:2.0:1775767798.449335:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00080000:3.0:1775767798.449336:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1775767798.449348:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1775767798.449349:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767798.449352:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767798.449353:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775767798.449355:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767798.449359:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767798.449360:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767798.449362:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767798.449368:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767798.449374:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767798.449377:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:3.0:1775767798.449380:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767798.449383:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:3.0:1775767798.449386:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:3.0:1775767798.449389:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:3.0:1775767798.449393:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767798.449396:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767798.449397:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767798.449399:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767798.449402:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:3.0:1775767798.449405:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767798.449408:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767798.449411:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767798.449415:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767798.449417:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767798.449420:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775767798.449422:0:36:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767798.449432:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8a80. 00000020:00000040:3.0:1775767798.449436:0:36:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:3.0:1775767798.449439:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767798.449441:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767798.449444:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767798.449447:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767798.449451:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767798.449460:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2800. 00000100:00000001:3.0:1775767798.449466:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767798.449474:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8a80 x1862022294236160/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767798.449484:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767798.449488:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767798.449493:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006c3265af] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8a80 x1862022294236160/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767798.449515:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767798.449519:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767798.449522:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775767798.449524:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767798.449526:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767798.449528:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767798.449531:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5050 00000400:00000200:3.0:1775767798.449536:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767798.449540:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767798.449543:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767798.449547:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767798.449550:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767798.449554:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767798.449559:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767798.449564:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767798.449567:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:3.0:1775767798.449571:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767798.449575:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:3.0:1775767798.449580:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:3.0:1775767798.449586:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:3.0:1775767798.449594:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767798.449599:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767798.449605:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767798.449611:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767798.449616:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00080000:3.0:1775767798.449628:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767798.449632:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767798.449634:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767798.449640:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767798.449643:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767798.449647:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767798.449651:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775767798.449655:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767798.449658:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767798.449663:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aa300. 00000020:00000040:3.0:1775767798.449667:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:3.0:1775767798.449672:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767798.449674:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767798.449678:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767798.449680:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767798.449683:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767798.449687:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0c00. 00000100:00000001:3.0:1775767798.449691:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767798.449699:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aa300 x1862022294236288/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767798.449710:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767798.449719:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767798.449727:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f33bdf15] to pc [ptlrpcd_rcv+-1] req@ffff9942099aa300 x1862022294236288/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767798.449753:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767798.449759:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767798.449767:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775767798.449770:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767798.449773:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767798.449786:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767798.449790:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5050 00000400:00000200:3.0:1775767798.449796:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767798.449803:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767798.449806:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767798.449812:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767798.449817:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:0.1:1775767798.449818:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000040:3.0:1775767798.449822:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000800:00000200:0.1:1775767798.449823:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000100:00000001:3.0:1775767798.449825:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767798.449828:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767798.449831:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:3.0:1775767798.449833:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767798.449837:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:3.0:1775767798.449842:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:3.0:1775767798.449856:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:3.0:1775767798.449866:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767798.449869:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767798.449873:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767798.449876:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767798.449878:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:3.0:1775767798.449884:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767798.449889:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767798.449893:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767798.449898:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767798.449900:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767798.449902:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767798.449906:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775767798.449909:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767798.449913:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767798.449917:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ab800. 00000020:00000040:3.0:1775767798.449921:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:3.0:1775767798.449924:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767798.449926:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767798.449930:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767798.449931:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767798.449932:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767798.449936:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2c00. 00000100:00000001:3.0:1775767798.449942:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767798.449948:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ab800 x1862022294236416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767798.449958:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767798.449960:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767798.449965:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000089df80c0] to pc [ptlrpcd_rcv+-1] req@ffff9942099ab800 x1862022294236416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767798.449979:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767798.449981:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767798.449985:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775767798.449989:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767798.449991:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767798.449995:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767798.450000:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5050 00000400:00000200:3.0:1775767798.450007:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767798.450013:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767798.450018:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767798.450023:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767798.450024:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767798.450028:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767798.450030:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767798.450034:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767798.450036:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:3.0:1775767798.450038:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767798.450042:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:3.0:1775767798.450044:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:3.0:1775767798.450046:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:3.0:1775767798.450052:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767798.450056:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767798.450060:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767798.450069:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767798.450074:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:3.0:1775767798.450080:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767798.450084:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767798.450087:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767798.450091:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767798.450094:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767798.450099:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767798.450101:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:3.0:1775767798.450103:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767798.450108:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767798.450113:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aa680. 00000020:00000040:3.0:1775767798.450116:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:3.0:1775767798.450121:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767798.450124:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767798.450128:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767798.450130:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767798.450133:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767798.450138:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0000. 00000100:00000001:3.0:1775767798.450143:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767798.450150:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aa680 x1862022294236544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767798.450182:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767798.450184:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767798.450188:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006cb1597d] to pc [ptlrpcd_rcv+-1] req@ffff9942099aa680 x1862022294236544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767798.450197:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767798.450200:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767798.450206:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775767798.450208:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767798.450211:0:36:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767798.450213:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767798.450218:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5050 00000400:00000200:3.0:1775767798.450225:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767798.450230:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767798.450234:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767798.450238:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767798.450241:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767798.450245:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767798.450250:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767798.450252:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767798.450255:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:3.0:1775767798.450257:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767798.450260:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:3.0:1775767798.450262:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:3.0:1775767798.450264:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:3.0:1775767798.450271:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767798.450275:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767798.450279:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767798.450283:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767798.450287:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:3.0:1775767798.450292:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767798.450307:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767798.450309:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767798.450313:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767798.450317:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767798.450322:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767798.450324:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:3.0:1775767798.450327:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767798.450330:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767798.450335:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aad80. 00000020:00000040:3.0:1775767798.450339:0:36:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:3.0:1775767798.450341:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767798.450344:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767798.450345:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767798.450348:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767798.450352:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767798.450359:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b3400. 00000100:00000001:3.0:1775767798.450364:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767798.450371:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aad80 x1862022294236672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767798.450380:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767798.450384:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767798.450388:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000045ccf08] to pc [ptlrpcd_rcv+-1] req@ffff9942099aad80 x1862022294236672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767798.450400:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767798.450405:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5060) 00000800:00000200:0.1:1775767798.450520:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767798.450522:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775767798.450742:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767798.450744:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775767798.450974:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767798.450975:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775767798.451292:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767798.451294:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000100:00000001:1.0:1775767799.473218:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767799.473224:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767799.473225:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767799.473230:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767799.473234:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767799.473237:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767799.473239:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767799.473241:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767799.473244:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767799.473273:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767799.473279:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767799.473280:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767799.473285:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767799.473289:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767799.473292:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767799.473294:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767799.473296:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767799.473298:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767799.473540:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767799.473544:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767799.473546:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767799.473550:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767799.473553:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767799.473556:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767799.473559:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767799.473560:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767799.473563:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767799.537433:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767799.537439:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767799.537442:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775767799.793256:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299724605, at 4299724608 00000100:00000010:1.0:1775767799.793270:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3000. 00000100:00000001:1.0:1775767800.497232:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767800.497232:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767800.497238:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767800.497239:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767800.497240:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767800.497241:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767800.497248:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775767800.497249:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767800.497256:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767800.497256:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767800.497261:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767800.497261:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767800.497264:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767800.497267:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767800.497270:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767800.497273:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767800.497275:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767800.497276:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767800.497282:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767800.497284:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767800.497285:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767800.497296:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767800.497301:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767800.497304:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767800.497307:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767800.497309:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767800.497312:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767801.521210:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767801.521217:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767801.521218:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767801.521220:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767801.521221:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767801.521222:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767801.521224:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767801.521226:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767801.521227:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767801.521229:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:2.0:1775767801.521232:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767801.521234:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767801.521236:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1775767801.521238:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767801.521239:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767801.521240:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767801.521246:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767801.521248:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767801.521249:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767801.521253:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767801.521254:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767801.521255:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767801.521258:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767801.521263:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767801.521266:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767801.521269:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767801.521272:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767801.585326:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767801.585329:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767801.585332:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775767802.545207:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767802.545213:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767802.545215:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767802.545218:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:3.0:1775767802.545220:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767802.545222:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767802.545225:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767802.545225:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1775767802.545229:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767802.545230:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767802.545234:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767802.545234:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767802.545238:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767802.545239:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767802.545243:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767802.545244:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767802.545246:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767802.545246:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767802.545249:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767802.545258:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767802.545260:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767802.545264:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767802.545269:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767802.545271:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767802.545274:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767802.545275:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767802.545277:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767803.569240:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767803.569248:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767803.569250:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767803.569252:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767803.569257:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:3.0:1775767803.569261:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767803.569266:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767803.569267:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767803.569276:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767803.569277:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767803.569284:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767803.569289:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767803.569291:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767803.569293:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767803.569297:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767803.569297:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767803.569303:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767803.569306:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767803.569311:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767803.569314:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767803.569316:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767803.569318:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767803.569324:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767803.569331:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767803.569337:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767803.569341:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767803.569352:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767803.569562:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767803.569567:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767803.569571:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767803.569580:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fc380 x1862022294236800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767803.569597:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767803.569600:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767803.569604:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767803.569608:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767803.569614:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767803.569620:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294236800:192.168.202.156@tcp:250:kworker.0 00000100:00000001:3.0:1775767803.569626:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767803.569632:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767803.569636:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767803.569640:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767803.569645:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b2000. 02000000:00000001:3.0:1775767803.569651:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767803.569660:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767803.569665:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767803.569672:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767803.569680:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294236800, portal 25 00000100:00000001:3.0:1775767803.569685:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767803.569688:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641197440 : -112966068354176 : ffff9942099fc380) 00000100:00000040:3.0:1775767803.569696:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fc380 x1862022294236800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767828 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767803.569709:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767803.569713:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767803.569716:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767803.569721:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294236800, offset 0 00000400:00000200:3.0:1775767803.569728:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767803.569741:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767803.569751:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767803.569759:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767803.569765:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767803.569783:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767803.569788:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767803.569809:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225023:225023:256:4294967295] 192.168.202.156@tcp LPNI seq info [225019:225019:8:4294967295] 00000400:00000200:3.0:1775767803.569818:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767803.569840:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767803.569850:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767803.569857:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767803.569868:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767803.569873:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775767803.569881:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767803.569884:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1775767803.569892:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767803.569897:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767803.569902:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767803.569911:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767803.569917:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767803.569920:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767803.569927:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767803.569933:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fc380 x1862022294236800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767828 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767803.569948:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767803.569953:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fc380 x1862022294236800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767828 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767803.569965:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767803.569967:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767803.569973:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767803.569979:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767803.569982:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299729725 00000100:00000010:3.0:1775767803.569988:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942079819c0. 00000100:00000001:3.0:1775767803.570008:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1775767803.570016:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299729725 00000100:00000010:3.0:1775767803.570021:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207981000. 00000100:00000001:3.0:1775767803.570027:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1775767803.570032:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767803.570035:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767803.570040:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767803.570047:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767803/real 1775767803] req@ffff9942099fc380 x1862022294236800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767803.570065:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767803.570068:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767803.570071:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767803.570077:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767803.570080:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767803.570085:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767803.570091:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fc380 x1862022294236800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767803.570111:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fc380 x1862022294236800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767803.570129:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767803.570132:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767803.570135:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767803.570138:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767803.570145:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fc380 x1862022294236800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767803.570178:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767803.570183:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fc380 x1862022294236800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767828 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767803.570189:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767803.570192:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767803.570194:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767803.570197:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767803.570204:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767803.570209:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767803.570213:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767803.570221:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767803.570225:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767803.570231:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fc380 x1862022294236800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767828 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767803.570242:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294236800:192.168.202.156@tcp:250:kworker.0 00000100:00000001:3.0:1775767803.570250:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767803.570253:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767803.570258:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fc380 x1862022294236800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767828 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767803.570276:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767803.570280:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767803.570285:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b2000. 02000000:00000001:3.0:1775767803.570289:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767803.570292:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767803.570294:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:3.0:1775767803.570299:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767803.570302:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002b800. 02000000:00000001:3.0:1775767803.570309:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767803.570312:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767803.570315:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fc380. 00000100:00000001:3.0:1775767803.570320:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767803.570323:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767803.570326:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767803.570334:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767803.570337:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767803.570341:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767803.570344:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767803.570346:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767803.570353:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fce00 x1862022294236928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767803.570369:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767803.570372:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767803.570375:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767803.570378:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767803.570382:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767803.570388:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294236928:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767803.570395:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767803.570398:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767803.570400:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767803.570403:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767803.570407:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b2000. 02000000:00000001:3.0:1775767803.570411:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767803.570416:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767803.570421:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767803.570425:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767803.570429:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294236928, portal 10 00000100:00000001:3.0:1775767803.570433:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767803.570436:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641200128 : -112966068351488 : ffff9942099fce00) 00000100:00000040:3.0:1775767803.570444:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fce00 x1862022294236928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767828 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767803.570456:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767803.570459:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767803.570464:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767803.570468:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294236928, offset 0 00000400:00000200:3.0:1775767803.570473:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767803.570480:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767803.570489:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767803.570495:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767803.570501:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767803.570508:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767803.570515:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767803.570524:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225024:225024:256:4294967295] 192.168.202.156@tcp LPNI seq info [225020:225020:8:4294967295] 00000400:00000200:3.0:1775767803.570538:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767803.570548:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767803.570553:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767803.570556:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767803.570561:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767803.570563:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775767803.570567:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767803.570568:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1775767803.570571:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767803.570572:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767803.570574:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767803.570577:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767803.570579:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767803.570580:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767803.570582:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767803.570585:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fce00 x1862022294236928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767828 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767803.570591:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767803.570593:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fce00 x1862022294236928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767828 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767803.570598:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767803.570599:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767803.570602:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767803.570605:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:3.0:1775767803.570607:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767803.570609:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299728403 00000100:00000001:3.0:1775767803.570611:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299729725 00000100:00000001:3.0:1775767803.570612:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299728403 00000100:00000001:3.0:1775767803.570614:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299729725 00000100:00000001:3.0:1775767803.570616:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767803.570617:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767803.570619:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767803.570621:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767803/real 1775767803] req@ffff9942099fce00 x1862022294236928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767803.570627:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767803.570627:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767803.570628:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767803.570630:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767803.570631:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767803.570632:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767803.570634:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fce00 x1862022294236928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767803.570639:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fce00 x1862022294236928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767803.570644:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767803.570644:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767803.570646:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767803.570646:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767803.570648:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fce00 x1862022294236928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767803.570652:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767803.570655:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fce00 x1862022294236928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767828 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767803.570659:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767803.570660:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767803.570661:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767803.570662:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767803.570663:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767803.570665:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767803.570668:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767803.570670:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767803.570672:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767803.570674:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fce00 x1862022294236928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767828 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767803.570679:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294236928:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767803.570682:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767803.570683:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fdc00 x1862022294237056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767803.570687:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767803.570688:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767803.570689:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767803.570691:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767803.570692:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767803.570694:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294237056:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767803.570696:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767803.570698:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767803.570699:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767803.570700:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767803.570703:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b1000. 02000000:00000001:3.0:1775767803.570704:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767803.570706:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767803.570708:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767803.570710:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767803.570712:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294237056, portal 10 00000100:00000001:3.0:1775767803.570713:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767803.570714:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641203712 : -112966068347904 : ffff9942099fdc00) 00000100:00000040:3.0:1775767803.570719:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fdc00 x1862022294237056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767828 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767803.570728:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767803.570729:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767803.570731:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767803.570732:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294237056, offset 0 00000400:00000200:3.0:1775767803.570735:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767803.570737:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767803.570739:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767803.570741:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767803.570743:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767803.570744:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767803.570746:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767803.570748:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225025:225025:256:4294967295] 192.168.202.156@tcp LPNI seq info [225021:225021:8:4294967295] 00000400:00000200:3.0:1775767803.570751:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767803.570755:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767803.570758:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767803.570761:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767803.570764:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767803.570766:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775767803.570768:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767803.570770:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1775767803.570772:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767803.570774:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767803.570776:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767803.570778:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767803.570780:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767803.570781:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767803.570782:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767803.570784:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fdc00 x1862022294237056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767828 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767803.570790:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767803.570791:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fdc00 x1862022294237056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767828 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767803.570817:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767803.570818:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767803.570821:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767803.570823:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:3.0:1775767803.570826:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767803.570827:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299728395 00000100:00000001:3.0:1775767803.570829:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299729725 00000100:00000001:3.0:1775767803.570831:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299728395 00000100:00000001:3.0:1775767803.570833:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299729725 00000100:00000001:3.0:1775767803.570834:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767803.570835:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767803.570837:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767803.570839:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767803/real 1775767803] req@ffff9942099fdc00 x1862022294237056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767803.570844:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767803.570845:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767803.570847:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767803.570848:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767803.570849:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767803.570851:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767803.570853:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fdc00 x1862022294237056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767803.570858:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fdc00 x1862022294237056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767803.570862:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767803.570863:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767803.570864:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767803.570865:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767803.570867:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fdc00 x1862022294237056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767803.570872:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767803.570875:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fdc00 x1862022294237056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767828 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767803.570879:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767803.570881:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767803.570882:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767803.570882:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767803.570883:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767803.570885:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767803.570887:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767803.570889:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767803.570890:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767803.570892:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fdc00 x1862022294237056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767828 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767803.570897:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294237056:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767803.570900:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767803.570902:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fe680 x1862022294237184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767803.570907:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767803.570908:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767803.570909:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767803.570910:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767803.570911:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767803.570913:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294237184:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767803.570915:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767803.570916:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767803.570917:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767803.570919:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767803.570921:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b0400. 02000000:00000001:3.0:1775767803.570922:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767803.570925:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767803.570927:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767803.570929:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767803.570931:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294237184, portal 4 00000100:00000001:3.0:1775767803.570932:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767803.570933:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641206400 : -112966068345216 : ffff9942099fe680) 00000100:00000040:3.0:1775767803.570937:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fe680 x1862022294237184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767828 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767803.570942:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767803.570943:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767803.570944:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767803.570946:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294237184, offset 0 00000400:00000200:3.0:1775767803.570948:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767803.570951:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767803.570953:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767803.570955:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767803.570956:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767803.570958:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767803.570960:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767803.570962:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225026:225026:256:4294967295] 192.168.202.156@tcp LPNI seq info [225022:225022:8:4294967295] 00000400:00000200:3.0:1775767803.570965:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767803.570970:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767803.570972:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767803.570974:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767803.570977:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767803.570979:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775767803.570982:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767803.570984:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1775767803.570987:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767803.570989:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767803.570991:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767803.570993:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767803.570995:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767803.570996:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767803.570998:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767803.571000:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fe680 x1862022294237184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767828 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767803.571005:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767803.571007:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fe680 x1862022294237184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767828 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767803.571011:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767803.571012:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767803.571013:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767803.571015:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:3.0:1775767803.571017:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767803.571019:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299728418 00000100:00000001:3.0:1775767803.571020:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299729725 00000100:00000001:3.0:1775767803.571022:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299728418 00000100:00000001:3.0:1775767803.571023:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299729725 00000100:00000001:3.0:1775767803.571024:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767803.571025:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767803.571027:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767803.571028:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767803/real 1775767803] req@ffff9942099fe680 x1862022294237184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767803.571033:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767803.571034:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767803.571034:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767803.571036:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767803.571037:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767803.571039:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767803.571041:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fe680 x1862022294237184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767803.571046:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fe680 x1862022294237184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767803.571050:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767803.571051:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767803.571052:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767803.571052:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767803.571054:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fe680 x1862022294237184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767803.571058:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767803.571060:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fe680 x1862022294237184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767828 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767803.571064:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767803.571066:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767803.571067:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767803.571067:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767803.571068:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767803.571071:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767803.571073:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767803.571074:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767803.571076:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767803.571078:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fe680 x1862022294237184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767828 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767803.571082:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294237184:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767803.571084:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767803.571086:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767803.571088:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fce00 x1862022294236928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767828 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767803.571093:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767803.571094:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767803.571095:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b2000. 02000000:00000001:3.0:1775767803.571097:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767803.571098:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767803.571098:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:3.0:1775767803.571100:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767803.571101:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002a800. 02000000:00000001:3.0:1775767803.571103:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767803.571104:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767803.571105:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fce00. 00000100:00000001:3.0:1775767803.571107:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767803.571108:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767803.571109:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767803.571111:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fdc00 x1862022294237056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767828 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767803.571116:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767803.571117:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767803.571117:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b1000. 02000000:00000001:3.0:1775767803.571119:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767803.571119:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767803.571120:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:3.0:1775767803.571121:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767803.571122:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029400. 02000000:00000001:3.0:1775767803.571123:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767803.571124:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767803.571125:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fdc00. 00000100:00000001:3.0:1775767803.571127:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767803.571128:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767803.571129:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767803.571131:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fe680 x1862022294237184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767828 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767803.571134:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767803.571136:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767803.571137:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b0400. 02000000:00000001:3.0:1775767803.571140:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767803.571141:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767803.571142:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:3.0:1775767803.571143:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767803.571144:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029800. 02000000:00000001:3.0:1775767803.571145:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767803.571146:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767803.571146:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fe680. 00000100:00000001:3.0:1775767803.571148:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767803.571148:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767803.571149:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767803.571152:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767803.571153:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767803.571170:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767803.571171:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767803.571172:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767803.571174:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fca80 x1862022294237312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767803.571179:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767803.571180:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767803.571181:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767803.571182:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767803.571184:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767803.571185:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294237312:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767803.571188:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767803.571189:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767803.571190:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767803.571191:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767803.571193:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b0400. 02000000:00000001:3.0:1775767803.571194:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767803.571197:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767803.571198:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767803.571200:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767803.571202:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294237312, portal 4 00000100:00000001:3.0:1775767803.571204:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767803.571205:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641199232 : -112966068352384 : ffff9942099fca80) 00000100:00000040:3.0:1775767803.571209:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fca80 x1862022294237312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767828 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767803.571213:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767803.571222:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767803.571224:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767803.571225:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294237312, offset 0 00000400:00000200:3.0:1775767803.571228:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767803.571230:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767803.571232:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767803.571235:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767803.571236:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767803.571238:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767803.571240:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767803.571242:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225027:225027:256:4294967295] 192.168.202.156@tcp LPNI seq info [225023:225023:8:4294967295] 00000400:00000200:3.0:1775767803.571245:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767803.571249:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767803.571252:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767803.571254:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767803.571256:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767803.571258:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775767803.571260:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767803.571262:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1775767803.571268:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767803.571270:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767803.571272:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767803.571274:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767803.571276:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767803.571277:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767803.571279:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767803.571281:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fca80 x1862022294237312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767828 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767803.571290:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767803.571292:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fca80 x1862022294237312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767828 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767803.571297:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767803.571298:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767803.571299:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767803.571301:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:3.0:1775767803.571303:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767803.571304:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299728422 00000100:00000001:3.0:1775767803.571306:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299729725 00000100:00000001:3.0:1775767803.571307:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299728422 00000100:00000001:3.0:1775767803.571309:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299729725 00000100:00000001:3.0:1775767803.571310:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767803.571312:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767803.571313:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767803.571315:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767803/real 1775767803] req@ffff9942099fca80 x1862022294237312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767803.571319:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767803.571320:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767803.571321:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767803.571323:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767803.571324:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767803.571325:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767803.571327:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fca80 x1862022294237312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767803.571332:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fca80 x1862022294237312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767803.571336:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767803.571337:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767803.571339:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767803.571340:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767803.571342:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fca80 x1862022294237312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767803.571346:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767803.571348:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fca80 x1862022294237312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767828 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767803.571352:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767803.571354:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767803.571355:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767803.571355:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767803.571356:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767803.571359:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767803.571361:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767803.571363:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767803.571365:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767803.571368:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fca80 x1862022294237312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767828 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767803.571373:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294237312:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767803.571375:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767803.571376:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767803.571378:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fca80 x1862022294237312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767828 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767803.571382:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767803.571383:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767803.571383:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b0400. 02000000:00000001:3.0:1775767803.571385:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767803.571385:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767803.571387:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:3.0:1775767803.571388:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767803.571389:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028000. 02000000:00000001:3.0:1775767803.571391:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767803.571392:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767803.571393:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fca80. 00000100:00000001:3.0:1775767803.571394:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767803.571395:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767803.571397:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767803.571399:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767803.571400:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767803.571401:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767803.571402:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767803.633358:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767803.633362:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767803.633367:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775767804.593210:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767804.593215:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767804.593217:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767804.593221:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767804.593225:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767804.593226:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767804.593228:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767804.593233:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767804.593233:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767804.593234:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767804.593236:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767804.593238:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767804.593241:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:3.0:1775767804.593241:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767804.593243:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767804.593244:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767804.593247:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:2.0:1775767804.593250:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767804.593252:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767804.593253:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767804.593256:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767804.593257:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767804.593260:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767804.593264:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767804.593264:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767804.593267:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767804.593270:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767804.913230:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299729725, at 4299729728 00000100:00000010:1.0:1775767804.913249:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207981000. 00000100:00000001:2.0:1775767805.617228:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767805.617232:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767805.617234:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767805.617237:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767805.617238:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767805.617241:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767805.617243:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:1.0:1775767805.617246:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775767805.617247:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767805.617251:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767805.617252:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767805.617256:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767805.617257:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767805.617258:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767805.617262:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767805.617262:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767805.617267:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767805.617274:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767805.681363:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767805.681367:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767805.681370:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775767806.641246:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767806.641252:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767806.641255:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767806.641256:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767806.641261:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:2.0:1775767806.641263:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767806.641264:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767806.641270:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775767806.641271:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767806.641273:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767806.641275:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767806.641277:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767806.641278:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767806.641281:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767806.641281:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767806.641285:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767806.641286:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767806.641287:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767806.641288:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767806.641290:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767806.641292:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767806.641294:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767806.641296:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767806.641300:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767806.641302:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767806.641304:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767806.641306:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767807.665225:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767807.665228:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767807.665231:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767807.665233:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767807.665234:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767807.665236:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767807.665241:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:1.0:1775767807.665242:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767807.665242:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767807.665244:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767807.665248:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767807.665250:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:2.0:1775767807.665250:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767807.665253:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767807.665255:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767807.665257:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767807.665257:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767807.665258:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767807.665263:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767807.665266:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767807.665268:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767807.665273:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767807.665275:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767807.665278:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767807.665278:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767807.665282:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767807.665285:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767807.729332:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767807.729335:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767807.729338:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775767808.689260:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767808.689260:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767808.689265:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767808.689268:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767808.689268:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767808.689271:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767808.689272:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767808.689272:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767808.689275:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767808.689281:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775767808.689281:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775767808.689281:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767808.689288:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767808.689289:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767808.689289:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767808.689295:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767808.689295:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767808.689296:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767808.689303:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767808.689308:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767808.689313:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767808.689314:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767808.689319:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767808.689325:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767808.689326:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767808.689331:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767808.689335:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767808.689350:0:9316:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767808.689357:0:9316:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775767808.689361:0:9316:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767808.689364:0:9316:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767808.689370:0:9316:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767808.689378:0:9316:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5061 00000400:00000200:3.0:1775767808.689389:0:9316:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767808.689396:0:9316:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767808.689402:0:9316:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767808.689408:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767808.689415:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767808.689429:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767808.689434:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767808.689440:0:9316:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767808.689444:0:9316:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:3.0:1775767808.689448:0:9316:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767808.689452:0:9316:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:3.0:1775767808.689456:0:9316:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:3.0:1775767808.689461:0:9316:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:3.0:1775767808.689468:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767808.689473:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767808.689478:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767808.689484:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767808.689491:0:9316:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:3.0:1775767808.689498:0:9316:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767808.689505:0:9316:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767808.689510:0:9316:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767808.689516:0:9316:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767808.689522:0:9316:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767808.689526:0:9316:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775767808.689530:0:9316:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767808.689540:0:9316:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8e00. 00000020:00000040:3.0:1775767808.689546:0:9316:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:3.0:1775767808.689553:0:9316:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767808.689557:0:9316:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767808.689562:0:9316:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767808.689566:0:9316:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767808.689572:0:9316:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767808.689582:0:9316:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0400. 00000100:00000001:3.0:1775767808.689592:0:9316:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767808.689604:0:9316:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8e00 x1862022294237440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767808.689623:0:9316:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767808.689630:0:9316:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767808.689639:0:9316:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000864a6866] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8e00 x1862022294237440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767808.689670:0:9316:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767808.689675:0:9316:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767808.689683:0:9316:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775767808.689688:0:9316:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767808.689690:0:9316:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767808.689697:0:9316:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767808.689703:0:9316:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5061 00000100:00000001:2.0:1775767808.689707:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775767808.689711:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000400:00000200:3.0:1775767808.689712:0:9316:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767808.689714:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000400:00000200:3.0:1775767808.689723:0:9316:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000040:2.0:1775767808.689725:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a8e00 x1862022294237440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767808.689727:0:9316:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767808.689736:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767808.689743:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775767808.689763:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767808.689767:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767808.689772:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:3.0:1775767808.689775:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000001:2.0:1775767808.689777:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767808.689780:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 02000000:00000001:2.0:1775767808.689783:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767808.689785:0:9316:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767808.689789:0:9316:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:3.0:1775767808.689792:0:9316:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00100000:2.0:1775767808.689794:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294237440:192.168.202.156@tcp:250:kworker.0 00000020:00000040:3.0:1775767808.689796:0:9316:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:3.0:1775767808.689802:0:9316:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000001:2.0:1775767808.689805:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000001:3.0:1775767808.689809:0:9316:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 02000000:00000001:2.0:1775767808.689812:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:3.0:1775767808.689816:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 02000000:00000001:2.0:1775767808.689818:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767808.689823:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1775767808.689824:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:3.0:1775767808.689828:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767808.689832:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 02000000:00000010:2.0:1775767808.689833:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1e400. 00000020:00000040:3.0:1775767808.689835:0:9316:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 02000000:00000001:2.0:1775767808.689838:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767808.689842:0:9316:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767808.689847:0:9316:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767808.689849:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:3.0:1775767808.689852:0:9316:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:2.0:1775767808.689858:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39c60. 02000000:00000001:3.0:1775767808.689859:0:9316:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767808.689864:0:9316:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775767808.689868:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f220. 00000100:00000001:3.0:1775767808.689869:0:9316:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767808.689875:0:9316:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00000200:2.0:1775767808.689879:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294237440, portal 25 00000008:00000001:3.0:1775767808.689882:0:9316:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767808.689884:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767808.689886:0:9316:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767808.689889:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640856064 : -112966068695552 : ffff9942099a8e00) 00000100:00000010:3.0:1775767808.689892:0:9316:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ab480. 00000020:00000040:3.0:1775767808.689897:0:9316:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000100:00000040:2.0:1775767808.689900:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a8e00 x1862022294237440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767833 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1775767808.689904:0:9316:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767808.689908:0:9316:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767808.689914:0:9316:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767808.689919:0:9316:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1775767808.689922:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1775767808.689923:0:9316:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767808.689930:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 02000000:00000010:3.0:1775767808.689932:0:9316:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b1000. 00000400:00000010:2.0:1775767808.689936:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000001:3.0:1775767808.689939:0:9316:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1775767808.689942:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294237440, offset 0 00000100:00100000:3.0:1775767808.689949:0:9316:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ab480 x1862022294237568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1775767808.689957:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00000001:3.0:1775767808.689967:0:9316:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:2.0:1775767808.689970:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767808.689972:0:9316:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767808.689979:0:9316:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000067a0ba74] to pc [ptlrpcd_rcv+-1] req@ffff9942099ab480 x1862022294237568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1775767808.689982:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767808.689993:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767808.690001:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000100:00000001:3.0:1775767808.690001:0:9316:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767808.690008:0:9316:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:2.0:1775767808.690011:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000100:00080000:3.0:1775767808.690016:0:9316:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775767808.690019:0:9316:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:2.0:1775767808.690021:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00080000:3.0:1775767808.690022:0:9316:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767808.690029:0:9316:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:2.0:1775767808.690030:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225028:225028:256:4294967295] 192.168.202.156@tcp LPNI seq info [225024:225024:8:4294967295] 00000100:00080000:3.0:1775767808.690035:0:9316:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5061 00000400:00000200:2.0:1775767808.690043:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767808.690051:0:9316:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767808.690059:0:9316:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767808.690063:0:9316:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1775767808.690065:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000100:00000040:3.0:1775767808.690069:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767808.690075:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767808.690078:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000800:00000200:2.0:1775767808.690084:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00000001:3.0:1775767808.690084:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767808.690086:0:9316:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767808.690089:0:9316:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:3.0:1775767808.690091:0:9316:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767808.690094:0:9316:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000800:00000010:2.0:1775767808.690095:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3300. 00000020:00000001:3.0:1775767808.690097:0:9316:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:3.0:1775767808.690102:0:9316:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:3.0:1775767808.690106:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000800:00000200:2.0:1775767808.690111:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:3.0:1775767808.690111:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767808.690115:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767808.690117:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000800:00000200:2.0:1775767808.690118:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000020:00000040:3.0:1775767808.690119:0:9316:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:3.0:1775767808.690125:0:9316:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767808.690131:0:9316:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775767808.690133:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 02000000:00000001:3.0:1775767808.690136:0:9316:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767808.690140:0:9316:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:2.1:1775767808.690141:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 02000000:00000001:3.0:1775767808.690146:0:9316:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767808.690151:0:9316:0:(obd_class.h:890:obd_reconnect()) Process entered 00000800:00000100:2.0:1775767808.690177:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000008:00100000:3.0:1775767808.690181:0:9316:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000010:2.0:1775767808.690182:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3300. 00000008:00000001:3.0:1775767808.690184:0:9316:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767808.690189:0:9316:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767808.690191:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000010:3.0:1775767808.690192:0:9316:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a9180. 00000020:00000040:3.0:1775767808.690197:0:9316:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:3.0:1775767808.690199:0:9316:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767808.690201:0:9316:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767808.690203:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000100:00000001:3.0:1775767808.690204:0:9316:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767808.690207:0:9316:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767808.690210:0:9316:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767808.690213:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 02000000:00000010:3.0:1775767808.690214:0:9316:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2000. 00000400:00000010:2.0:1775767808.690219:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:3.0:1775767808.690219:0:9316:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767808.690227:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00100000:3.0:1775767808.690227:0:9316:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a9180 x1862022294237696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:2.0:1775767808.690235:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a8e00 x1862022294237440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767833 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767808.690237:0:9316:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767808.690241:0:9316:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767808.690248:0:9316:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bdc2586] to pc [ptlrpcd_rcv+-1] req@ffff9942099a9180 x1862022294237696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767808.690258:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1775767808.690260:0:9316:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767808.690263:0:9316:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775767808.690267:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a8e00 x1862022294237440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767833 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767808.690267:0:9316:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775767808.690269:0:9316:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767808.690271:0:9316:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767808.690275:0:9316:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767808.690280:0:9316:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5061 00000400:00000200:3.0:1775767808.690287:0:9316:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767808.690291:0:9316:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:2.0:1775767808.690293:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767808.690293:0:9316:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1775767808.690298:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:3.0:1775767808.690298:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767808.690302:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767808.690303:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767808.690305:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:2.0:1775767808.690306:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767808.690306:0:9316:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767808.690308:0:9316:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:3.0:1775767808.690310:0:9316:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767808.690312:0:9316:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:3.0:1775767808.690315:0:9316:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000400:00000200:2.0:1775767808.690316:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:3.0:1775767808.690318:0:9316:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:2.0:1775767808.690321:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299734846 00000100:00000040:3.0:1775767808.690322:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767808.690324:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767808.690327:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767808.690328:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000010:2.0:1775767808.690329:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d6120. 00000020:00000040:3.0:1775767808.690330:0:9316:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:3.0:1775767808.690334:0:9316:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767808.690336:0:9316:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767808.690337:0:9316:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767808.690339:0:9316:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767808.690340:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 02000000:00000001:3.0:1775767808.690341:0:9316:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767808.690345:0:9316:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767808.690347:0:9316:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000100:00000001:2.0:1775767808.690350:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299734846 00000008:00000001:3.0:1775767808.690350:0:9316:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767808.690352:0:9316:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767808.690357:0:9316:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8700. 00000100:00000010:2.0:1775767808.690359:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d66c0. 00000020:00000040:3.0:1775767808.690359:0:9316:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:3.0:1775767808.690362:0:9316:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767808.690364:0:9316:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767808.690365:0:9316:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1775767808.690366:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 02000000:00000001:3.0:1775767808.690368:0:9316:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767808.690370:0:9316:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767808.690373:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000010:3.0:1775767808.690374:0:9316:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b3400. 00000100:00000001:3.0:1775767808.690379:0:9316:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767808.690380:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767808.690383:0:9316:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8700 x1862022294237824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775767808.690385:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1775767808.690391:0:9316:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767808.690394:0:9316:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767808.690396:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767808/real 1775767808] req@ffff9942099a8e00 x1862022294237440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775767808.690399:0:9316:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000faa25e3b] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8700 x1862022294237824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767808.690428:0:9316:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767808.690434:0:9316:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767808.690441:0:9316:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775767808.690444:0:9316:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767808.690448:0:9316:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000001:2.0:1775767808.690451:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1775767808.690451:0:9316:0:(import.c:484:import_select_connection()) Process entered 00000400:00000001:2.0:1775767808.690455:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:3.0:1775767808.690456:0:9316:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5061 00000400:00000010:2.0:1775767808.690460:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39c60. 00000400:00000200:3.0:1775767808.690464:0:9316:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767808.690466:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f220 00000400:00000200:3.0:1775767808.690470:0:9316:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000400:00000010:2.0:1775767808.690471:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f220. 00000100:00080000:3.0:1775767808.690473:0:9316:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1775767808.690476:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:3.0:1775767808.690479:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767808.690482:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:2.0:1775767808.690484:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a8e00 x1862022294237440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775767808.690487:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767808.690492:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767808.690497:0:9316:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767808.690499:0:9316:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:3.0:1775767808.690503:0:9316:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767808.690505:0:9316:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000100:00000200:2.0:1775767808.690509:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a8e00 x1862022294237440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775767808.690509:0:9316:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:3.0:1775767808.690514:0:9316:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:3.0:1775767808.690522:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767808.690528:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775767808.690531:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000040:3.0:1775767808.690532:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767808.690535:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767808.690536:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:2.0:1775767808.690541:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000040:3.0:1775767808.690541:0:9316:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00000001:2.0:1775767808.690545:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775767808.690547:0:9316:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767808.690552:0:9316:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767808.690554:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a8e00 x1862022294237440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1775767808.690556:0:9316:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767808.690561:0:9316:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767808.690566:0:9316:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767808.690568:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767808.690569:0:9316:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767808.690574:0:9316:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000100:00000040:2.0:1775767808.690577:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a8e00 x1862022294237440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767833 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1775767808.690578:0:9316:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767808.690583:0:9316:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767808.690590:0:9316:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8380. 00000100:00000001:2.0:1775767808.690593:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000040:3.0:1775767808.690594:0:9316:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:3.0:1775767808.690600:0:9316:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1775767808.690601:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1775767808.690603:0:9316:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767808.690604:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767808.690606:0:9316:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1775767808.690607:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1775767808.690608:0:9316:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1775767808.690611:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 02000000:00000001:3.0:1775767808.690612:0:9316:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767808.690614:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000010:3.0:1775767808.690618:0:9316:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0000. 00000100:00080000:2.0:1775767808.690619:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1775767808.690624:0:9316:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775767808.690626:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767808.690630:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1775767808.690632:0:9316:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8380 x1862022294237952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:2.0:1775767808.690639:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a8e00 x1862022294237440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767833 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767808.690669:0:9316:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767808.690672:0:9316:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767808.690673:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294237440:192.168.202.156@tcp:250:kworker.0 00000100:00000040:3.0:1775767808.690681:0:9316:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055f5621a] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8380 x1862022294237952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767808.690682:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767808.690688:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1775767808.690695:0:9316:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767808.690696:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a8e00 x1862022294237440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767833 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775767808.690699:0:9316:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5071) 00000100:00000001:2.0:1775767808.690708:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767808.690710:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767808.690714:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1e400. 02000000:00000001:2.0:1775767808.690718:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767808.690720:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767808.690722:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:2.0:1775767808.690725:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767808.690727:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b0400. 02000000:00000001:2.0:1775767808.690731:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767808.690733:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767808.690735:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a8e00. 00000100:00000001:2.0:1775767808.690738:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767808.690739:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767808.690742:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767808.690763:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767808.690764:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767808.690766:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775767808.690768:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767808.690770:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767808.690774:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ab480 x1862022294237568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767808.690783:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767808.690784:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767808.690787:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767808.690789:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767808.690791:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767808.690794:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ab480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294237568:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775767808.690798:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767808.690800:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767808.690801:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767808.690803:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775767808.690807:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1e400. 02000000:00000001:2.0:1775767808.690809:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767808.690813:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767808.690816:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39c60. 00000400:00000010:2.0:1775767808.690820:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f220. 00000100:00000200:2.0:1775767808.690823:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294237568, portal 10 00000100:00000001:2.0:1775767808.690825:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767808.690827:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640865920 : -112966068685696 : ffff9942099ab480) 00000100:00000040:2.0:1775767808.690833:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ab480 x1862022294237568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767833 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767808.690840:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767808.690842:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767808.690845:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775767808.690848:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294237568, offset 0 00000400:00000200:2.0:1775767808.690852:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767808.690857:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767808.690861:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767808.690864:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767808.690867:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767808.690870:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767808.690873:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767808.690876:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225029:225029:256:4294967295] 192.168.202.156@tcp LPNI seq info [225025:225025:8:4294967295] 00000400:00000200:2.0:1775767808.690880:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767808.690887:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767808.690892:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767808.690896:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3300. 00000800:00000200:2.0:1775767808.690900:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775767808.690903:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775767808.690908:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767808.690911:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000100:2.0:1775767808.690915:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767808.690917:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3300. 00000400:00000200:2.0:1775767808.690919:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775767808.690923:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775767808.690927:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775767808.690929:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775767808.690931:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767808.690935:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ab480 x1862022294237568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767833 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767808.690943:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767808.690946:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ab480 x1862022294237568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767833 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767808.690953:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767808.690955:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775767808.690958:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767808.690961:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:2.0:1775767808.690965:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767808.690967:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299733523 00000100:00000001:2.0:1775767808.690970:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299734846 00000100:00000001:2.0:1775767808.690972:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299733523 00000100:00000001:2.0:1775767808.690974:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299734846 00000100:00000001:2.0:1775767808.690977:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767808.690979:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767808.690981:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767808.690985:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767808/real 1775767808] req@ffff9942099ab480 x1862022294237568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775767808.690993:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767808.690995:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767808.690997:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39c60. 00000400:00000200:2.0:1775767808.691000:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f220 00000400:00000010:2.0:1775767808.691001:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f220. 00000100:00000001:2.0:1775767808.691004:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767808.691007:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ab480 x1862022294237568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767808.691016:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ab480 x1862022294237568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767808.691023:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767808.691024:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767808.691026:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767808.691027:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767808.691031:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ab480 x1862022294237568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767808.691039:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767808.691042:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ab480 x1862022294237568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767833 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767808.691050:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767808.691053:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767808.691054:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767808.691056:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767808.691058:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767808.691060:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767808.691063:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767808.691067:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767808.691069:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767808.691073:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ab480 x1862022294237568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767833 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767808.691082:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ab480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294237568:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775767808.691087:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767808.691090:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a9180 x1862022294237696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767808.691098:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767808.691099:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767808.691102:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767808.691104:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767808.691106:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767808.691109:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294237696:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775767808.691112:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767808.691114:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767808.691116:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767808.691118:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775767808.691121:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1fc00. 02000000:00000001:2.0:1775767808.691123:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767808.691126:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767808.691129:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39c60. 00000400:00000010:2.0:1775767808.691132:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f220. 00000100:00000200:2.0:1775767808.691135:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294237696, portal 10 00000100:00000001:2.0:1775767808.691138:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767808.691139:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640856960 : -112966068694656 : ffff9942099a9180) 00000100:00000040:2.0:1775767808.691145:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a9180 x1862022294237696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767833 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767808.691151:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767808.691171:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767808.691175:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775767808.691178:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294237696, offset 0 00000400:00000200:2.0:1775767808.691182:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767808.691187:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767808.691191:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767808.691194:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767808.691197:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767808.691199:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767808.691202:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767808.691205:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225030:225030:256:4294967295] 192.168.202.156@tcp LPNI seq info [225026:225026:8:4294967295] 00000400:00000200:2.0:1775767808.691210:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767808.691217:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767808.691221:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767808.691225:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3300. 00000800:00000200:2.0:1775767808.691229:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775767808.691232:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775767808.691236:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767808.691239:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000100:2.0:1775767808.691243:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767808.691245:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3300. 00000400:00000200:2.0:1775767808.691248:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775767808.691251:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775767808.691254:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775767808.691256:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775767808.691258:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767808.691262:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a9180 x1862022294237696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767833 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767808.691270:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767808.691272:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a9180 x1862022294237696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767833 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767808.691279:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767808.691281:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775767808.691284:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767808.691287:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:2.0:1775767808.691290:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767808.691292:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299733516 00000100:00000001:2.0:1775767808.691295:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299734846 00000100:00000001:2.0:1775767808.691297:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299733516 00000100:00000001:2.0:1775767808.691300:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299734846 00000100:00000001:2.0:1775767808.691302:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767808.691304:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767808.691306:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767808.691310:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767808/real 1775767808] req@ffff9942099a9180 x1862022294237696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775767808.691318:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767808.691320:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767808.691321:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39c60. 00000400:00000200:2.0:1775767808.691323:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f220 00000400:00000010:2.0:1775767808.691325:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f220. 00000100:00000001:2.0:1775767808.691327:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767808.691330:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a9180 x1862022294237696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767808.691339:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a9180 x1862022294237696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767808.691346:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767808.691347:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767808.691349:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767808.691351:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767808.691354:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a9180 x1862022294237696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767808.691362:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767808.691365:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a9180 x1862022294237696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767833 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767808.691372:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767808.691375:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767808.691377:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767808.691378:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767808.691380:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767808.691383:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767808.691386:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767808.691389:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767808.691392:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767808.691396:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a9180 x1862022294237696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767833 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767808.691405:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294237696:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775767808.691409:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767808.691413:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a8700 x1862022294237824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767808.691420:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767808.691421:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767808.691424:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767808.691426:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767808.691428:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767808.691431:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294237824:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767808.691434:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767808.691436:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767808.691438:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767808.691439:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775767808.691443:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1d400. 02000000:00000001:2.0:1775767808.691445:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767808.691449:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767808.691452:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39c60. 00000400:00000010:2.0:1775767808.691455:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f220. 00000100:00000200:2.0:1775767808.691458:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294237824, portal 4 00000100:00000001:2.0:1775767808.691460:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767808.691461:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640854272 : -112966068697344 : ffff9942099a8700) 00000100:00000040:2.0:1775767808.691467:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a8700 x1862022294237824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767833 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767808.691473:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767808.691476:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767808.691478:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775767808.691481:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294237824, offset 0 00000400:00000200:2.0:1775767808.691486:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767808.691490:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767808.691494:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767808.691497:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767808.691500:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767808.691503:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767808.691505:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767808.691508:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225031:225031:256:4294967295] 192.168.202.156@tcp LPNI seq info [225027:225027:8:4294967295] 00000400:00000200:2.0:1775767808.691513:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767808.691520:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767808.691525:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767808.691528:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3300. 00000800:00000200:2.0:1775767808.691532:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775767808.691535:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775767808.691539:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767808.691541:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000100:2.0:1775767808.691545:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767808.691547:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3300. 00000400:00000200:2.0:1775767808.691550:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775767808.691553:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775767808.691556:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775767808.691558:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775767808.691561:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767808.691565:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a8700 x1862022294237824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767833 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767808.691573:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767808.691577:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a8700 x1862022294237824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767833 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767808.691590:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767808.691592:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775767808.691595:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767808.691598:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:2.0:1775767808.691603:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767808.691605:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299733539 00000100:00000001:2.0:1775767808.691607:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299734846 00000100:00000001:2.0:1775767808.691610:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299733539 00000100:00000001:2.0:1775767808.691612:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299734846 00000100:00000001:2.0:1775767808.691614:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767808.691616:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767808.691618:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767808.691622:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767808/real 1775767808] req@ffff9942099a8700 x1862022294237824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775767808.691631:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767808.691632:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767808.691634:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39c60. 00000400:00000200:2.0:1775767808.691636:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f220 00000400:00000010:2.0:1775767808.691638:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f220. 00000100:00000001:2.0:1775767808.691641:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767808.691644:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a8700 x1862022294237824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767808.691653:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a8700 x1862022294237824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767808.691660:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767808.691662:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767808.691664:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767808.691665:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767808.691669:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a8700 x1862022294237824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767808.691676:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767808.691680:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a8700 x1862022294237824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767833 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767808.691688:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767808.691691:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767808.691693:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767808.691694:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767808.691695:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767808.691699:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767808.691701:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767808.691705:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767808.691708:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767808.691712:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a8700 x1862022294237824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767833 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767808.691720:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294237824:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767808.691725:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767808.691728:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a8380 x1862022294237952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767808.691735:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767808.691736:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767808.691738:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767808.691740:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767808.691742:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767808.691762:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294237952:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767808.691766:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767808.691768:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767808.691770:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767808.691772:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775767808.691775:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f800. 02000000:00000001:2.0:1775767808.691777:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767808.691780:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767808.691783:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39c60. 00000400:00000010:2.0:1775767808.691786:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f220. 00000100:00000200:2.0:1775767808.691789:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294237952, portal 4 00000100:00000001:2.0:1775767808.691791:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767808.691792:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640853376 : -112966068698240 : ffff9942099a8380) 00000100:00000040:2.0:1775767808.691797:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a8380 x1862022294237952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767833 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767808.691805:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767808.691807:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767808.691810:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775767808.691812:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294237952, offset 0 00000400:00000200:2.0:1775767808.691817:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767808.691821:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767808.691824:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767808.691827:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767808.691829:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767808.691832:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767808.691835:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767808.691838:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225032:225032:256:4294967295] 192.168.202.156@tcp LPNI seq info [225028:225028:8:4294967295] 00000400:00000200:2.0:1775767808.691842:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767808.691849:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767808.691853:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767808.691856:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3300. 00000800:00000200:2.0:1775767808.691860:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775767808.691863:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775767808.691868:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767808.691870:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000100:2.0:1775767808.691874:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767808.691877:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3300. 00000400:00000200:2.0:1775767808.691879:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775767808.691883:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775767808.691886:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775767808.691888:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775767808.691890:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767808.691894:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a8380 x1862022294237952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767833 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767808.691902:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767808.691905:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a8380 x1862022294237952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767833 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767808.691912:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767808.691913:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775767808.691916:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767808.691919:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:2.0:1775767808.691922:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767808.691924:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299733542 00000100:00000001:2.0:1775767808.691927:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299734846 00000100:00000001:2.0:1775767808.691930:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299733542 00000100:00000001:2.0:1775767808.691932:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299734846 00000100:00000001:2.0:1775767808.691934:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767808.691936:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767808.691938:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767808.691941:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767808/real 1775767808] req@ffff9942099a8380 x1862022294237952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775767808.691950:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767808.691951:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767808.691953:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39c60. 00000400:00000200:2.0:1775767808.691955:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f220 00000400:00000010:2.0:1775767808.691957:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f220. 00000100:00000001:2.0:1775767808.691960:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767808.691963:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a8380 x1862022294237952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767808.691973:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a8380 x1862022294237952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767808.691980:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767808.691982:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767808.691984:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767808.691985:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767808.691988:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a8380 x1862022294237952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767808.691996:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767808.692000:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a8380 x1862022294237952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767833 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767808.692007:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767808.692009:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767808.692011:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767808.692012:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767808.692014:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767808.692018:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767808.692021:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767808.692024:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767808.692026:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767808.692030:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a8380 x1862022294237952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767833 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767808.692040:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294237952:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767808.692044:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767808.692046:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767808.692049:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ab480 x1862022294237568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767833 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767808.692056:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767808.692058:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767808.692060:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1e400. 02000000:00000001:2.0:1775767808.692063:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767808.692064:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767808.692066:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:2.0:1775767808.692068:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767808.692070:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b1000. 02000000:00000001:2.0:1775767808.692073:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767808.692074:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767808.692076:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ab480. 00000100:00000001:2.0:1775767808.692079:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767808.692080:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767808.692083:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767808.692086:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a9180 x1862022294237696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767833 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767808.692093:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767808.692094:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767808.692095:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1fc00. 02000000:00000001:2.0:1775767808.692098:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767808.692099:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767808.692100:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:2.0:1775767808.692102:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767808.692104:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b2000. 02000000:00000001:2.0:1775767808.692106:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767808.692107:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767808.692109:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a9180. 00000100:00000001:2.0:1775767808.692111:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767808.692113:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767808.692115:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767808.692118:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a8700 x1862022294237824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767833 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767808.692125:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767808.692126:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767808.692128:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1d400. 02000000:00000001:2.0:1775767808.692132:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767808.692133:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767808.692134:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:2.0:1775767808.692136:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767808.692137:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b3400. 02000000:00000001:2.0:1775767808.692139:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767808.692140:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767808.692142:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a8700. 00000100:00000001:2.0:1775767808.692144:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767808.692145:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767808.692147:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767808.692151:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a8380 x1862022294237952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767833 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767808.692175:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767808.692177:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767808.692179:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f800. 02000000:00000001:2.0:1775767808.692182:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767808.692183:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767808.692184:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:2.0:1775767808.692186:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767808.692187:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b0000. 02000000:00000001:2.0:1775767808.692189:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767808.692190:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767808.692192:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a8380. 00000100:00000001:2.0:1775767808.692194:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767808.692195:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767808.692197:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767808.692201:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767808.692202:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767808.692204:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775767808.692206:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767809.713228:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767809.713229:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767809.713231:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767809.713235:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767809.713236:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767809.713237:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767809.713240:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767809.713243:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:1.0:1775767809.713244:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775767809.713244:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775767809.713245:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767809.713250:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767809.713251:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767809.713251:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767809.713254:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767809.713256:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767809.713257:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767809.713257:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767809.713260:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767809.713262:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767809.713264:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767809.713268:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767809.713270:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767809.713274:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767809.713274:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767809.713276:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767809.713278:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767809.777374:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767809.777377:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767809.777380:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775767810.033247:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299734846, at 4299734848 00000100:00000010:1.0:1775767810.033261:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d66c0. 00000100:00000001:3.0:1775767810.737235:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767810.737238:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767810.737240:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767810.737242:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767810.737243:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767810.737244:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767810.737248:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:1.0:1775767810.737250:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775767810.737250:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775767810.737253:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767810.737256:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767810.737256:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767810.737257:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767810.737261:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767810.737262:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767810.737262:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767810.737266:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767810.737269:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767810.737269:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767810.737273:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767810.737273:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767810.737274:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767810.737277:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767810.737281:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767810.737281:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767810.737284:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767810.737287:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767811.761272:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767811.761274:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767811.761276:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767811.761278:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767811.761280:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767811.761281:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767811.761282:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767811.761284:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767811.761287:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767811.761287:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:2.0:1775767811.761289:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767811.761294:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1775767811.761295:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775767811.761295:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767811.761300:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767811.761300:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767811.761301:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767811.761306:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767811.761306:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767811.761309:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767811.761312:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767811.761313:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767811.761317:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767811.761320:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767811.761320:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767811.761324:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767811.761329:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767811.825351:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767811.825355:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767811.825358:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775767812.785303:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767812.785303:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767812.785312:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767812.785314:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767812.785315:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767812.785320:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767812.785321:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767812.785329:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1775767812.785331:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767812.785336:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767812.785338:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767812.785340:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767812.785342:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767812.785343:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767812.785357:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767812.785358:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767812.785364:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767812.785375:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767813.809267:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767813.809267:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767813.809273:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767813.809274:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767813.809276:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767813.809280:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767813.809283:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767813.809287:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767813.809288:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767813.809289:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00100000:1.0:1775767813.809293:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767813.809294:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767813.809299:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767813.809299:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767813.809304:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767813.809306:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767813.809307:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767813.809309:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767813.809449:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767813.809454:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767813.809458:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767813.809463:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767813.809468:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767813.809473:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767813.809477:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767813.809480:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767813.809484:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767813.809612:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775767813.809618:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767813.809622:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767813.809633:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fea00 x1862022294238080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767813.809650:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767813.809652:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767813.809658:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767813.809661:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767813.809665:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767813.809670:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294238080:192.168.202.156@tcp:250:kworker.0 00000100:00000001:2.0:1775767813.809674:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767813.809678:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767813.809680:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767813.809682:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775767813.809687:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1d400. 02000000:00000001:2.0:1775767813.809692:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767813.809710:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767813.809715:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39c60. 00000400:00000010:2.0:1775767813.809722:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f220. 00000100:00000200:2.0:1775767813.809730:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294238080, portal 25 00000100:00000001:2.0:1775767813.809734:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767813.809739:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641207296 : -112966068344320 : ffff9942099fea00) 00000100:00000040:2.0:1775767813.809749:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fea00 x1862022294238080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767838 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000001:3.0:1775767813.809755:0:9316:0:(rw.c:1481:ll_writepages()) Process entered 00000080:00000001:3.0:1775767813.809759:0:9316:0:(file.c:5250:cl_sync_file_range()) Process entered 00000020:00000001:3.0:1775767813.809763:0:9316:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00000001:2.0:1775767813.809768:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000010:3.0:1775767813.809770:0:9316:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff99420693e000. 00000100:00000040:2.0:1775767813.809776:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000020:00000010:3.0:1775767813.809776:0:9316:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dce600. 00000400:00000010:2.0:1775767813.809784:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000020:00000010:3.0:1775767813.809784:0:9316:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210dcea00. 00000020:00000010:3.0:1775767813.809790:0:9316:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207981f00. 00000100:00000200:2.0:1775767813.809792:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294238080, offset 0 00000008:00000010:3.0:1775767813.809797:0:9316:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994208b50000. 00020000:00000010:3.0:1775767813.809802:0:9316:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994218000300. 00000400:00000200:2.0:1775767813.809806:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00008000:00000010:3.0:1775767813.809809:0:9316:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994207e08e20. 00000080:00000010:3.0:1775767813.809815:0:9316:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99420541b1f8. 00000400:00000200:2.0:1775767813.809819:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000080:00000010:3.0:1775767813.809821:0:9316:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077c0248. 00000020:00000010:3.0:1775767813.809826:0:9316:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dce200. 00000400:00000200:2.0:1775767813.809831:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000008:00000010:3.0:1775767813.809831:0:9316:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942086f8420. 00020000:00000010:3.0:1775767813.809837:0:9316:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994207e8e400. 00008000:00000010:3.0:1775767813.809841:0:9316:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994209270a90. 00000400:00000200:2.0:1775767813.809842:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000080:00000010:3.0:1775767813.809845:0:9316:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942090e33c0. 00000020:00000001:3.0:1775767813.809849:0:9316:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107590086664 : -112966119464952 : ffff99420693e008) 00000400:00000200:2.0:1775767813.809852:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000020:00001000:3.0:1775767813.809853:0:9316:0:(cl_object.c:831:cl_env_get()) 1@ffff99420693e000 00000400:00000200:2.0:1775767813.809859:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000020:00000001:3.0:1775767813.809859:0:9316:0:(cl_object.c:151:cl_object_top()) ffff9942078a76c0 -> ffff9942078a76c0 00000020:00000001:3.0:1775767813.809864:0:9316:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:3.0:1775767813.809866:0:9316:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000400:00000200:2.0:1775767813.809867:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000080:00200000:3.0:1775767813.809870:0:9316:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:3.0:1775767813.809876:0:9316:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000400:00000200:2.0:1775767813.809877:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225033:225033:256:4294967295] 192.168.202.156@tcp LPNI seq info [225029:225029:8:4294967295] 00000020:00000001:3.0:1775767813.809877:0:9316:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:3.0:1775767813.809880:0:9316:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:3.0:1775767813.809886:0:9316:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017a:0x0]io ffff9942077c02a8 type 6 ignore/verify layout 1/0 00000400:00000200:2.0:1775767813.809890:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000002:3.0:1775767813.809891:0:9316:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207c1f9b0) owner 0000000000000000/ffff994201384f80 00020000:00000001:3.0:1775767813.809896:0:9316:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:3.0:1775767813.809898:0:9316:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:3.0:1775767813.809901:0:9316:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00020000:00000001:3.0:1775767813.809903:0:9316:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767813.809908:0:9316:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:3.0:1775767813.809912:0:9316:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00000400:00000200:2.0:1775767813.809913:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00020000:00000001:3.0:1775767813.809914:0:9316:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767813.809917:0:9316:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:3.0:1775767813.809920:0:9316:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:3.0:1775767813.809922:0:9316:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00020000:00000002:3.0:1775767813.809924:0:9316:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207c1f9b0) owner 0000000000000000/ffff994201384f80 00000020:00000001:3.0:1775767813.809928:0:9316:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1775767813.809929:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000020:00000001:3.0:1775767813.809931:0:9316:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:3.0:1775767813.809934:0:9316:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00020000:00000001:3.0:1775767813.809936:0:9316:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00000800:00000010:2.0:1775767813.809939:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3300. 00020000:00200000:3.0:1775767813.809939:0:9316:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00000002:3.0:1775767813.809945:0:9316:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:3.0:1775767813.809949:0:9316:0:(lov_io.c:155:lov_sub_get()) Process entered 00020000:00000001:3.0:1775767813.809952:0:9316:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000800:00000200:2.0:1775767813.809953:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000001:3.0:1775767813.809954:0:9316:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:3.0:1775767813.809958:0:9316:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff99420693ecf0. 00000800:00000200:2.0:1775767813.809960:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000020:00000010:3.0:1775767813.809962:0:9316:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dce000. 00000020:00000010:3.0:1775767813.809967:0:9316:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210dcf400. 00000020:00000010:3.0:1775767813.809970:0:9316:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207981180. 00000800:00000200:2.1:1775767813.809972:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000008:00000010:3.0:1775767813.809974:0:9316:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994208b51650. 00020000:00000010:3.0:1775767813.809978:0:9316:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994218000000. 00000800:00000200:2.1:1775767813.809979:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00008000:00000010:3.0:1775767813.809981:0:9316:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994207e0ade8. 00000080:00000010:3.0:1775767813.809986:0:9316:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99420541b0a8. 00000080:00000010:3.0:1775767813.809989:0:9316:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077c1da8. 00000800:00000100:2.0:1775767813.809990:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000020:00000010:3.0:1775767813.809994:0:9316:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcf000. 00000800:00000010:2.0:1775767813.809995:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3300. 00000008:00000010:3.0:1775767813.809999:0:9316:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942086f8160. 00020000:00000010:3.0:1775767813.810002:0:9316:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994207e8ed00. 00000400:00000200:2.0:1775767813.810003:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00008000:00000010:3.0:1775767813.810006:0:9316:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994209270ef0. 00000080:00000010:3.0:1775767813.810010:0:9316:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942090e3960. 00000020:00000001:3.0:1775767813.810014:0:9316:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107590089976 : -112966119461640 : ffff99420693ecf8) 00000400:00000200:2.0:1775767813.810015:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000020:00001000:3.0:1775767813.810019:0:9316:0:(cl_object.c:831:cl_env_get()) 1@ffff99420693ecf0 00000020:00000001:3.0:1775767813.810022:0:9316:0:(cl_object.c:151:cl_object_top()) ffff9942078a7788 -> ffff9942078a76c0 00000400:00000200:2.0:1775767813.810023:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000020:00000001:3.0:1775767813.810025:0:9316:0:(cl_io.c:125:__cl_io_init()) Process entered 00000400:00000010:2.0:1775767813.810028:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000020:00000001:3.0:1775767813.810029:0:9316:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:3.0:1775767813.810031:0:9316:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:3.0:1775767813.810034:0:9316:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767813.810036:0:2399:0:(events.c:34:request_out_callback()) Process entered 00020000:00000001:3.0:1775767813.810037:0:9316:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767813.810040:0:9316:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107612435560 : -112966097116056 : ffff994207e8e468) 00000100:00000200:2.0:1775767813.810045:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fea00 x1862022294238080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767838 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775767813.810045:0:9316:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000020:00000001:3.0:1775767813.810047:0:9316:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767813.810050:0:9316:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767813.810054:0:9316:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767813.810058:0:9316:0:(cl_io.c:314:cl_io_lock()) Process entered 00020000:00000001:3.0:1775767813.810061:0:9316:0:(lov_io.c:1108:lov_io_lock()) Process entered 00020000:00000001:3.0:1775767813.810064:0:9316:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775767813.810066:0:9316:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:3.0:1775767813.810068:0:9316:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:3.0:1775767813.810071:0:9316:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767813.810075:0:9316:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767813.810077:0:9316:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767813.810078:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00020000:00000001:3.0:1775767813.810079:0:9316:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767813.810082:0:9316:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:3.0:1775767813.810084:0:9316:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767813.810087:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fea00 x1862022294238080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767838 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775767813.810087:0:9316:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767813.810089:0:9316:0:(cl_io.c:528:cl_io_start()) Process entered 00020000:00000001:3.0:1775767813.810092:0:9316:0:(lov_io.c:1114:lov_io_start()) Process entered 00020000:00000001:3.0:1775767813.810094:0:9316:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775767813.810096:0:9316:0:(cl_io.c:528:cl_io_start()) Process entered 00000008:00000001:3.0:1775767813.810098:0:9316:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000008:00000001:3.0:1775767813.810102:0:9316:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000008:00000001:3.0:1775767813.810108:0:9316:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:3.0:1775767813.810111:0:9316:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:3.0:1775767813.810114:0:9316:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767813.810115:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1775767813.810117:0:9316:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000008:00000001:3.0:1775767813.810120:0:9316:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000100:00000001:2.0:1775767813.810121:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000008:00000001:3.0:1775767813.810123:0:9316:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107884691392 : -112965824860224 : ffff994218232fc0) 00000100:00000001:2.0:1775767813.810128:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00000020:3.0:1775767813.810128:0:9316:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994218232fc0 ready 0|-|- wr 1|-|+ rd 0|- 10 in flight 00000008:00000001:3.0:1775767813.810134:0:9316:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:3.0:1775767813.810136:0:9316:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000400:00000200:2.0:1775767813.810138:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00000001:3.0:1775767813.810138:0:9316:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1775767813.810144:0:9316:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000100:00000001:2.0:1775767813.810145:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299739965 00000008:00000020:3.0:1775767813.810146:0:9316:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994210675880 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000008:00000001:3.0:1775767813.810152:0:9316:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1775767813.810177:0:9316:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767813.810179:0:9316:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767813.810181:0:9316:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775767813.810184:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d66c0. 00020000:00000001:3.0:1775767813.810184:0:9316:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767813.810186:0:9316:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767813.810189:0:9316:0:(cl_io.c:555:cl_io_end()) Process entered 00020000:00000001:3.0:1775767813.810192:0:9316:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00020000:00000001:3.0:1775767813.810193:0:9316:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000100:00000001:2.0:1775767813.810194:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000020:00000001:3.0:1775767813.810195:0:9316:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000001:3.0:1775767813.810197:0:9316:0:(cl_io.c:563:cl_io_end()) Process leaving 00020000:00000001:3.0:1775767813.810199:0:9316:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767813.810202:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299739966 00020000:00000001:3.0:1775767813.810202:0:9316:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000020:00000001:3.0:1775767813.810204:0:9316:0:(cl_io.c:563:cl_io_end()) Process leaving 00000020:00000001:3.0:1775767813.810208:0:9316:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000100:00000010:2.0:1775767813.810210:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d6120. 00020000:00000001:3.0:1775767813.810211:0:9316:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00020000:00000001:3.0:1775767813.810214:0:9316:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775767813.810216:0:9316:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000100:00000001:2.0:1775767813.810217:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000020:00000001:3.0:1775767813.810218:0:9316:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:3.0:1775767813.810220:0:9316:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767813.810222:0:9316:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767813.810225:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:3.0:1775767813.810225:0:9316:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000020:00000001:3.0:1775767813.810228:0:9316:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:3.0:1775767813.810230:0:9316:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000020:00000001:3.0:1775767813.810232:0:9316:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000100:00000001:2.0:1775767813.810233:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767813.810234:0:9316:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:3.0:1775767813.810237:0:9316:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00020000:00000001:3.0:1775767813.810238:0:9316:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000001:2.0:1775767813.810239:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000020:00000001:3.0:1775767813.810240:0:9316:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000001:3.0:1775767813.810243:0:9316:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00020000:00000001:3.0:1775767813.810245:0:9316:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767813.810247:0:9316:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767813.810249:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767813/real 1775767813] req@ffff9942099fea00 x1862022294238080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:3.0:1775767813.810250:0:9316:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000020:00000001:3.0:1775767813.810252:0:9316:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000020:00000001:3.0:1775767813.810255:0:9316:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767813.810259:0:9316:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:3.0:1775767813.810261:0:9316:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:3.0:1775767813.810264:0:9316:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:3.0:1775767813.810266:0:9316:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:3.0:1775767813.810269:0:9316:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:3.0:1775767813.810272:0:9316:0:(cl_object.c:925:cl_env_put()) 1@ffff99420693ecf0 00000020:00000010:3.0:1775767813.810276:0:9316:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210dcf400. 00000020:00000010:3.0:1775767813.810283:0:9316:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207981180. 00000400:00000001:2.0:1775767813.810284:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000008:00000010:3.0:1775767813.810288:0:9316:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994208b51650. 00020000:00000010:3.0:1775767813.810292:0:9316:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994218000000. 00008000:00000010:3.0:1775767813.810295:0:9316:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994207e0ade8. 00000400:00000001:2.0:1775767813.810297:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000080:00000010:3.0:1775767813.810299:0:9316:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99420541b0a8. 00000400:00000010:2.0:1775767813.810301:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39c60. 00000080:00000010:3.0:1775767813.810303:0:9316:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077c1da8. 00000020:00000010:3.0:1775767813.810308:0:9316:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dce000. 00000400:00000200:2.0:1775767813.810309:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f220 00000008:00000010:3.0:1775767813.810312:0:9316:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942086f8160. 00000400:00000010:2.0:1775767813.810315:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f220. 00020000:00000010:3.0:1775767813.810316:0:9316:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994207e8ed00. 00000100:00000001:2.0:1775767813.810321:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00008000:00000010:3.0:1775767813.810322:0:9316:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994209270ef0. 00000080:00000010:3.0:1775767813.810327:0:9316:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942090e3960. 00000100:00000200:2.0:1775767813.810331:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fea00 x1862022294238080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1775767813.810331:0:9316:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcf000. 00000020:00000010:3.0:1775767813.810335:0:9316:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff99420693ecf0. 00020000:00000001:3.0:1775767813.810338:0:9316:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:3.0:1775767813.810341:0:9316:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:3.0:1775767813.810345:0:9316:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000080:00200000:3.0:1775767813.810348:0:9316:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:3.0:1775767813.810355:0:9316:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:3.0:1775767813.810357:0:9316:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:3.0:1775767813.810359:0:9316:0:(cl_object.c:925:cl_env_put()) 1@ffff99420693e000 00000100:00000200:2.0:1775767813.810361:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fea00 x1862022294238080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1775767813.810362:0:9316:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210dcea00. 00000020:00000010:3.0:1775767813.810366:0:9316:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207981f00. 00000008:00000010:3.0:1775767813.810370:0:9316:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994208b50000. 00020000:00000010:3.0:1775767813.810374:0:9316:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994218000300. 00008000:00000010:3.0:1775767813.810377:0:9316:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994207e08e20. 00000080:00000010:3.0:1775767813.810382:0:9316:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99420541b1f8. 00000080:00000010:3.0:1775767813.810386:0:9316:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077c0248. 00000100:00000001:2.0:1775767813.810389:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000010:3.0:1775767813.810389:0:9316:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dce600. 00000100:00000001:2.0:1775767813.810394:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00000010:3.0:1775767813.810394:0:9316:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942086f8420. 00020000:00000010:3.0:1775767813.810398:0:9316:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994207e8e400. 00000100:00000001:2.0:1775767813.810400:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00008000:00000010:3.0:1775767813.810401:0:9316:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994209270a90. 00000100:00000001:2.0:1775767813.810405:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000080:00000010:3.0:1775767813.810406:0:9316:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942090e33c0. 00000020:00000010:3.0:1775767813.810410:0:9316:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dce200. 00000020:00000010:3.0:1775767813.810413:0:9316:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff99420693e000. 00000100:00100000:2.0:1775767813.810416:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fea00 x1862022294238080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000001:3.0:1775767813.810416:0:9316:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:3.0:1775767813.810421:0:9316:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:3.0:1775767813.810429:0:9316:0:(rw.c:1481:ll_writepages()) Process entered 00000080:00000001:3.0:1775767813.810431:0:9316:0:(file.c:5250:cl_sync_file_range()) Process entered 00000020:00000001:3.0:1775767813.810434:0:9316:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00000001:2.0:1775767813.810435:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:3.0:1775767813.810437:0:9316:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff99420693e000. 00000020:00000010:3.0:1775767813.810442:0:9316:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dce200. 00000020:00000010:3.0:1775767813.810445:0:9316:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210dce600. 00000100:00000040:2.0:1775767813.810448:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fea00 x1862022294238080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767838 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1775767813.810448:0:9316:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207981f00. 00000008:00000010:3.0:1775767813.810453:0:9316:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994208b50000. 00020000:00000010:3.0:1775767813.810457:0:9316:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994218000300. 00008000:00000010:3.0:1775767813.810461:0:9316:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994207e08e20. 00000080:00000010:3.0:1775767813.810466:0:9316:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99420541b1f8. 00000080:00000010:3.0:1775767813.810469:0:9316:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077c0248. 00000020:00000010:3.0:1775767813.810473:0:9316:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcea00. 00000008:00000010:3.0:1775767813.810476:0:9316:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942086f8420. 00000100:00000001:2.0:1775767813.810480:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00020000:00000010:3.0:1775767813.810481:0:9316:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994207e8e400. 00008000:00000010:3.0:1775767813.810485:0:9316:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994209270a90. 00000100:00000001:2.0:1775767813.810487:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000080:00000010:3.0:1775767813.810489:0:9316:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942090e33c0. 00000100:00000001:2.0:1775767813.810493:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:3.0:1775767813.810494:0:9316:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107590086664 : -112966119464952 : ffff99420693e008) 00000100:00000001:2.0:1775767813.810497:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00001000:3.0:1775767813.810498:0:9316:0:(cl_object.c:831:cl_env_get()) 1@ffff99420693e000 00000100:00000001:2.0:1775767813.810502:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000020:00000001:3.0:1775767813.810502:0:9316:0:(cl_object.c:151:cl_object_top()) ffff9942078a7f58 -> ffff9942078a7f58 00000100:00000001:2.0:1775767813.810506:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:3.0:1775767813.810506:0:9316:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:3.0:1775767813.810508:0:9316:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:3.0:1775767813.810510:0:9316:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017b:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000100:00080000:2.0:1775767813.810512:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000020:00000001:3.0:1775767813.810515:0:9316:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:3.0:1775767813.810518:0:9316:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:3.0:1775767813.810520:0:9316:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775767813.810521:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00020000:00000002:3.0:1775767813.810526:0:9316:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017b:0x0]io ffff9942077c02a8 type 6 ignore/verify layout 1/0 00000100:00000001:2.0:1775767813.810527:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00020000:00000002:3.0:1775767813.810533:0:9316:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207c1f000) owner 0000000000000000/ffff994201384f80 00000100:00000040:2.0:1775767813.810537:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fea00 x1862022294238080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767838 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:3.0:1775767813.810538:0:9316:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:3.0:1775767813.810541:0:9316:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:3.0:1775767813.810544:0:9316:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00020000:00000001:3.0:1775767813.810547:0:9316:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767813.810551:0:9316:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:3.0:1775767813.810556:0:9316:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00000100:00100000:2.0:1775767813.810557:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294238080:192.168.202.156@tcp:250:kworker.0 00020000:00000001:3.0:1775767813.810557:0:9316:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767813.810561:0:9316:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:3.0:1775767813.810563:0:9316:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:3.0:1775767813.810565:0:9316:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00000100:00000001:2.0:1775767813.810567:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00020000:00000002:3.0:1775767813.810567:0:9316:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207c1f000) owner 0000000000000000/ffff994201384f80 00000020:00000001:3.0:1775767813.810569:0:9316:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767813.810571:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000020:00000001:3.0:1775767813.810572:0:9316:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:3.0:1775767813.810574:0:9316:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00020000:00000001:3.0:1775767813.810575:0:9316:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00020000:00200000:3.0:1775767813.810578:0:9316:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00000040:2.0:1775767813.810579:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fea00 x1862022294238080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767838 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000002:3.0:1775767813.810580:0:9316:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:3.0:1775767813.810585:0:9316:0:(lov_io.c:155:lov_sub_get()) Process entered 00020000:00000001:3.0:1775767813.810586:0:9316:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000020:00000001:3.0:1775767813.810588:0:9316:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:3.0:1775767813.810592:0:9316:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff99420693ecf0. 00000100:00000001:2.0:1775767813.810596:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 00000020:00000010:3.0:1775767813.810596:0:9316:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcf000. 00000020:00000010:3.0:1775767813.810600:0:9316:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210dce000. 02000000:00000001:2.0:1775767813.810601:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000010:3.0:1775767813.810602:0:9316:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207981180. 02000000:00000010:2.0:1775767813.810606:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1d400. 00000008:00000010:3.0:1775767813.810606:0:9316:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994208b51650. 00020000:00000010:3.0:1775767813.810607:0:9316:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994218000000. 00008000:00000010:3.0:1775767813.810611:0:9316:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994207e0ade8. 02000000:00000001:2.0:1775767813.810613:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000080:00000010:3.0:1775767813.810613:0:9316:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99420541b0a8. 00000020:00000001:2.0:1775767813.810616:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000080:00000010:3.0:1775767813.810617:0:9316:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077c1da8. 00000020:00000040:2.0:1775767813.810620:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000010:3.0:1775767813.810620:0:9316:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcf400. 00000008:00000010:3.0:1775767813.810624:0:9316:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942086f8160. 00000020:00000001:2.0:1775767813.810625:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 00020000:00000010:3.0:1775767813.810626:0:9316:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994207e8ed00. 02000000:00000010:2.0:1775767813.810629:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028400. 02000000:00000001:2.0:1775767813.810634:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767813.810635:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00008000:00000010:3.0:1775767813.810637:0:9316:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994209270ef0. 00000100:00000010:2.0:1775767813.810638:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fea00. 00000080:00000010:3.0:1775767813.810640:0:9316:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942090e3960. 00000100:00000001:2.0:1775767813.810643:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000020:00000001:3.0:1775767813.810644:0:9316:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107590089976 : -112966119461640 : ffff99420693ecf8) 00000100:00000001:2.0:1775767813.810646:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00001000:3.0:1775767813.810647:0:9316:0:(cl_object.c:831:cl_env_get()) 1@ffff99420693ecf0 00000020:00000001:3.0:1775767813.810649:0:9316:0:(cl_object.c:151:cl_object_top()) ffff9942078a75f8 -> ffff9942078a7f58 00000100:00000001:2.0:1775767813.810651:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1775767813.810652:0:9316:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:3.0:1775767813.810654:0:9316:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:3.0:1775767813.810656:0:9316:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:3.0:1775767813.810657:0:9316:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767813.810658:0:9316:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767813.810660:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00020000:00000001:3.0:1775767813.810660:0:9316:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107612435560 : -112966097116056 : ffff994207e8e468) 00000100:00000001:2.0:1775767813.810663:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767813.810664:0:9316:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000020:00000001:3.0:1775767813.810665:0:9316:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767813.810669:0:9316:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767813.810670:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1775767813.810670:0:9316:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767813.810673:0:9316:0:(cl_io.c:314:cl_io_lock()) Process entered 00000100:00000001:2.0:1775767813.810674:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00020000:00000001:3.0:1775767813.810675:0:9316:0:(lov_io.c:1108:lov_io_lock()) Process entered 00020000:00000001:3.0:1775767813.810677:0:9316:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000001:2.0:1775767813.810678:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1775767813.810678:0:9316:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:3.0:1775767813.810681:0:9316:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:3.0:1775767813.810682:0:9316:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767813.810683:0:9316:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767813.810686:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fca80 x1862022294238208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:3.0:1775767813.810686:0:9316:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767813.810687:0:9316:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767813.810689:0:9316:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:3.0:1775767813.810690:0:9316:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767813.810693:0:9316:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767813.810706:0:9316:0:(cl_io.c:528:cl_io_start()) Process entered 00020000:00000001:3.0:1775767813.810707:0:9316:0:(lov_io.c:1114:lov_io_start()) Process entered 00020000:00000001:3.0:1775767813.810709:0:9316:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775767813.810711:0:9316:0:(cl_io.c:528:cl_io_start()) Process entered 00000008:00000001:3.0:1775767813.810713:0:9316:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000008:00000001:3.0:1775767813.810716:0:9316:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000008:00000001:3.0:1775767813.810720:0:9316:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:3.0:1775767813.810722:0:9316:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:3.0:1775767813.810724:0:9316:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1775767813.810729:0:9316:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000008:00000001:3.0:1775767813.810731:0:9316:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000100:00000001:2.0:1775767813.810733:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000008:00000001:3.0:1775767813.810733:0:9316:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107884688256 : -112965824863360 : ffff994218232380) 00000100:00000001:2.0:1775767813.810736:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000020:3.0:1775767813.810737:0:9316:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994218232380 ready 0|-|- wr 1|-|+ rd 0|- 9 in flight 00000008:00000001:3.0:1775767813.810742:0:9316:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 02000000:00000001:2.0:1775767813.810743:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000020:3.0:1775767813.810744:0:9316:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:3.0:1775767813.810746:0:9316:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1775767813.810748:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:3.0:1775767813.810748:0:9316:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000008:00000020:3.0:1775767813.810750:0:9316:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994210674fc0 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 02000000:00000001:2.0:1775767813.810755:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1775767813.810755:0:9316:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1775767813.810758:0:9316:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767813.810760:0:9316:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767813.810763:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294238208:192.168.202.156@tcp:38:kworker.0 00020000:00000001:3.0:1775767813.810763:0:9316:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767813.810764:0:9316:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767813.810767:0:9316:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767813.810770:0:9316:0:(cl_io.c:555:cl_io_end()) Process entered 00020000:00000001:3.0:1775767813.810772:0:9316:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00000100:00000001:2.0:1775767813.810773:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00020000:00000001:3.0:1775767813.810774:0:9316:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000020:00000001:3.0:1775767813.810777:0:9316:0:(cl_io.c:555:cl_io_end()) Process entered 02000000:00000001:2.0:1775767813.810778:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:3.0:1775767813.810778:0:9316:0:(cl_io.c:563:cl_io_end()) Process leaving 00020000:00000001:3.0:1775767813.810779:0:9316:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767813.810782:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767813.810782:0:9316:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000020:00000001:3.0:1775767813.810783:0:9316:0:(cl_io.c:563:cl_io_end()) Process leaving 00000020:00000001:3.0:1775767813.810784:0:9316:0:(cl_io.c:352:cl_io_unlock()) Process entered 02000000:00000001:2.0:1775767813.810787:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00020000:00000001:3.0:1775767813.810787:0:9316:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00020000:00000001:3.0:1775767813.810789:0:9316:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775767813.810791:0:9316:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000020:00000001:3.0:1775767813.810791:0:9316:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:3.0:1775767813.810792:0:9316:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775767813.810793:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1d400. 00020000:00000001:3.0:1775767813.810794:0:9316:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767813.810797:0:9316:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 02000000:00000001:2.0:1775767813.810799:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767813.810799:0:9316:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:3.0:1775767813.810800:0:9316:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000020:00000001:3.0:1775767813.810802:0:9316:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000001:3.0:1775767813.810804:0:9316:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:3.0:1775767813.810805:0:9316:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00020000:00000001:3.0:1775767813.810806:0:9316:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775767813.810808:0:9316:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000400:00000200:2.0:1775767813.810809:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:3.0:1775767813.810809:0:9316:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00020000:00000001:3.0:1775767813.810811:0:9316:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767813.810813:0:9316:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775767813.810815:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39c60. 00020000:00000001:3.0:1775767813.810815:0:9316:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000020:00000001:3.0:1775767813.810816:0:9316:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000020:00000001:3.0:1775767813.810818:0:9316:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767813.810820:0:9316:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:3.0:1775767813.810821:0:9316:0:(lov_io.c:675:lov_io_fini()) Process entered 00000400:00000010:2.0:1775767813.810822:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f220. 00020000:00000001:3.0:1775767813.810822:0:9316:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:3.0:1775767813.810823:0:9316:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:3.0:1775767813.810825:0:9316:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:3.0:1775767813.810827:0:9316:0:(cl_object.c:925:cl_env_put()) 1@ffff99420693ecf0 00000100:00000200:2.0:1775767813.810829:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294238208, portal 10 00000020:00000010:3.0:1775767813.810830:0:9316:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210dce000. 00000020:00000010:3.0:1775767813.810834:0:9316:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207981180. 00000100:00000001:2.0:1775767813.810835:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767813.810838:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641199232 : -112966068352384 : ffff9942099fca80) 00000008:00000010:3.0:1775767813.810839:0:9316:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994208b51650. 00020000:00000010:3.0:1775767813.810842:0:9316:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994218000000. 00008000:00000010:3.0:1775767813.810844:0:9316:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994207e0ade8. 00000080:00000010:3.0:1775767813.810848:0:9316:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99420541b0a8. 00000100:00000040:2.0:1775767813.810850:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fca80 x1862022294238208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767838 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000010:3.0:1775767813.810850:0:9316:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077c1da8. 00000020:00000010:3.0:1775767813.810854:0:9316:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcf000. 00000008:00000010:3.0:1775767813.810857:0:9316:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942086f8160. 00020000:00000010:3.0:1775767813.810861:0:9316:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994207e8ed00. 00008000:00000010:3.0:1775767813.810865:0:9316:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994209270ef0. 00000080:00000010:3.0:1775767813.810868:0:9316:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942090e3960. 00000100:00000001:2.0:1775767813.810871:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000010:3.0:1775767813.810871:0:9316:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcf400. 00000020:00000010:3.0:1775767813.810876:0:9316:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff99420693ecf0. 00020000:00000001:3.0:1775767813.810879:0:9316:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00000100:00000040:2.0:1775767813.810880:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00020000:00000001:3.0:1775767813.810881:0:9316:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:3.0:1775767813.810883:0:9316:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000400:00000010:2.0:1775767813.810884:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000080:00200000:3.0:1775767813.810885:0:9316:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:3.0:1775767813.810889:0:9316:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000100:00000200:2.0:1775767813.810890:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294238208, offset 0 00000020:00000001:3.0:1775767813.810890:0:9316:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:3.0:1775767813.810892:0:9316:0:(cl_object.c:925:cl_env_put()) 1@ffff99420693e000 00000020:00000010:3.0:1775767813.810895:0:9316:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210dce600. 00000020:00000010:3.0:1775767813.810898:0:9316:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207981f00. 00000008:00000010:3.0:1775767813.810901:0:9316:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994208b50000. 00000400:00000200:2.0:1775767813.810902:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00020000:00000010:3.0:1775767813.810903:0:9316:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994218000300. 00008000:00000010:3.0:1775767813.810906:0:9316:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994207e08e20. 00000080:00000010:3.0:1775767813.810908:0:9316:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99420541b1f8. 00000400:00000200:2.0:1775767813.810912:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000080:00000010:3.0:1775767813.810912:0:9316:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077c0248. 00000020:00000010:3.0:1775767813.810915:0:9316:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dce200. 00000008:00000010:3.0:1775767813.810917:0:9316:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942086f8420. 00020000:00000010:3.0:1775767813.810920:0:9316:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994207e8e400. 00000400:00000200:2.0:1775767813.810921:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00008000:00000010:3.0:1775767813.810923:0:9316:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994209270a90. 00000400:00000200:2.0:1775767813.810926:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000080:00000010:3.0:1775767813.810926:0:9316:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942090e33c0. 00000020:00000010:3.0:1775767813.810929:0:9316:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcea00. 00000400:00000200:2.0:1775767813.810933:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000020:00000010:3.0:1775767813.810933:0:9316:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff99420693e000. 00000080:00000001:3.0:1775767813.810937:0:9316:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1775767813.810938:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000080:00000001:3.0:1775767813.810939:0:9316:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767813.810944:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000080:00000001:3.0:1775767813.810945:0:9316:0:(rw.c:1481:ll_writepages()) Process entered 00000080:00000001:3.0:1775767813.810947:0:9316:0:(file.c:5250:cl_sync_file_range()) Process entered 00000020:00000001:3.0:1775767813.810949:0:9316:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:3.0:1775767813.810952:0:9316:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff99420693e000. 00000400:00000200:2.0:1775767813.810953:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225034:225034:256:4294967295] 192.168.202.156@tcp LPNI seq info [225030:225030:8:4294967295] 00000020:00000010:3.0:1775767813.810956:0:9316:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcea00. 00000020:00000010:3.0:1775767813.810959:0:9316:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210dce200. 00000020:00000010:3.0:1775767813.810962:0:9316:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207981f00. 00000400:00000200:2.0:1775767813.810963:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00000010:3.0:1775767813.810965:0:9316:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994208b50000. 00020000:00000010:3.0:1775767813.810968:0:9316:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994218000300. 00008000:00000010:3.0:1775767813.810971:0:9316:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994207e08e20. 00000080:00000010:3.0:1775767813.810975:0:9316:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99420541b1f8. 00000400:00000200:2.0:1775767813.810979:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000080:00000010:3.0:1775767813.810979:0:9316:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077c0248. 00000020:00000010:3.0:1775767813.810982:0:9316:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dce600. 00000008:00000010:3.0:1775767813.810986:0:9316:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942086f8420. 00020000:00000010:3.0:1775767813.810989:0:9316:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994207e8e400. 00000800:00000200:2.0:1775767813.810990:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00008000:00000010:3.0:1775767813.810991:0:9316:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994209270a90. 00000080:00000010:3.0:1775767813.810996:0:9316:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942090e33c0. 00000800:00000010:2.0:1775767813.810997:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3300. 00000020:00000001:3.0:1775767813.810998:0:9316:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107590086664 : -112966119464952 : ffff99420693e008) 00000020:00001000:3.0:1775767813.811002:0:9316:0:(cl_object.c:831:cl_env_get()) 1@ffff99420693e000 00000020:00000001:3.0:1775767813.811004:0:9316:0:(cl_object.c:151:cl_object_top()) ffff99420791a850 -> ffff99420791a850 00000020:00000001:3.0:1775767813.811006:0:9316:0:(cl_io.c:125:__cl_io_init()) Process entered 00000800:00000200:2.0:1775767813.811008:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000080:00000001:3.0:1775767813.811008:0:9316:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:3.0:1775767813.811010:0:9316:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017f:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:3.0:1775767813.811014:0:9316:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000800:00000200:2.0:1775767813.811015:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000020:00000001:3.0:1775767813.811015:0:9316:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:3.0:1775767813.811016:0:9316:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:3.0:1775767813.811019:0:9316:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017f:0x0]io ffff9942077c02a8 type 6 ignore/verify layout 1/0 00020000:00000002:3.0:1775767813.811022:0:9316:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207dfbd90) owner 0000000000000000/ffff994201384f80 00020000:00000001:3.0:1775767813.811026:0:9316:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00000800:00000200:2.1:1775767813.811027:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00020000:00000001:3.0:1775767813.811028:0:9316:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:3.0:1775767813.811030:0:9316:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00020000:00000001:3.0:1775767813.811033:0:9316:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775767813.811033:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00020000:00000001:3.0:1775767813.811034:0:9316:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:3.0:1775767813.811038:0:9316:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:3.0:1775767813.811040:0:9316:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767813.811044:0:9316:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000800:00000100:2.0:1775767813.811045:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000020:00000001:3.0:1775767813.811045:0:9316:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:3.0:1775767813.811047:0:9316:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00020000:00000002:3.0:1775767813.811049:0:9316:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207dfbd90) owner 0000000000000000/ffff994201384f80 00000800:00000010:2.0:1775767813.811050:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3300. 00000020:00000001:3.0:1775767813.811051:0:9316:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767813.811054:0:9316:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:3.0:1775767813.811055:0:9316:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000400:00000200:2.0:1775767813.811057:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00020000:00000001:3.0:1775767813.811057:0:9316:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00020000:00200000:3.0:1775767813.811059:0:9316:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00000002:3.0:1775767813.811062:0:9316:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:3.0:1775767813.811065:0:9316:0:(lov_io.c:155:lov_sub_get()) Process entered 00000400:00000200:2.0:1775767813.811066:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00020000:00000001:3.0:1775767813.811067:0:9316:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000020:00000001:3.0:1775767813.811069:0:9316:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:3.0:1775767813.811071:0:9316:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff99420693ecf0. 00000400:00000200:2.0:1775767813.811074:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000020:00000010:3.0:1775767813.811074:0:9316:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcf400. 00000020:00000010:3.0:1775767813.811078:0:9316:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210dcf000. 00000400:00000010:2.0:1775767813.811079:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000020:00000010:3.0:1775767813.811080:0:9316:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207981180. 00000100:00000001:2.0:1775767813.811084:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000008:00000010:3.0:1775767813.811084:0:9316:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994208b51650. 00020000:00000010:3.0:1775767813.811085:0:9316:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994218000000. 00008000:00000010:3.0:1775767813.811088:0:9316:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994207e0ade8. 00000100:00000200:2.0:1775767813.811091:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fca80 x1862022294238208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767838 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000010:3.0:1775767813.811091:0:9316:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99420541b0a8. 00000080:00000010:3.0:1775767813.811092:0:9316:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077c1da8. 00000020:00000010:3.0:1775767813.811096:0:9316:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dce000. 00000008:00000010:3.0:1775767813.811099:0:9316:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942086f8160. 00020000:00000010:3.0:1775767813.811102:0:9316:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994207e8ed00. 00008000:00000010:3.0:1775767813.811106:0:9316:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994209270ef0. 00000080:00000010:3.0:1775767813.811109:0:9316:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942090e3960. 00000020:00000001:3.0:1775767813.811112:0:9316:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107590089976 : -112966119461640 : ffff99420693ecf8) 00000100:00000001:2.0:1775767813.811114:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000020:00001000:3.0:1775767813.811114:0:9316:0:(cl_object.c:831:cl_env_get()) 1@ffff99420693ecf0 00000020:00000001:3.0:1775767813.811117:0:9316:0:(cl_object.c:151:cl_object_top()) ffff99420791a210 -> ffff99420791a850 00000020:00000001:3.0:1775767813.811119:0:9316:0:(cl_io.c:125:__cl_io_init()) Process entered 00000100:00000040:2.0:1775767813.811122:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fca80 x1862022294238208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767838 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775767813.811122:0:9316:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:3.0:1775767813.811124:0:9316:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:3.0:1775767813.811126:0:9316:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767813.811129:0:9316:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767813.811132:0:9316:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107612435560 : -112966097116056 : ffff994207e8e468) 00000020:00000001:3.0:1775767813.811135:0:9316:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000020:00000001:3.0:1775767813.811137:0:9316:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767813.811141:0:9316:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767813.811143:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767813.811143:0:9316:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767813.811147:0:9316:0:(cl_io.c:314:cl_io_lock()) Process entered 00000100:00000001:2.0:1775767813.811148:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00020000:00000001:3.0:1775767813.811149:0:9316:0:(lov_io.c:1108:lov_io_lock()) Process entered 00020000:00000001:3.0:1775767813.811150:0:9316:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775767813.811152:0:9316:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:3.0:1775767813.811153:0:9316:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000100:00000001:2.0:1775767813.811175:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767813.811178:0:9316:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767813.811180:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000020:00000001:3.0:1775767813.811182:0:9316:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767813.811186:0:9316:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767813.811190:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:3.0:1775767813.811191:0:9316:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767813.811195:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299738644 00000020:00000001:3.0:1775767813.811195:0:9316:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:3.0:1775767813.811198:0:9316:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767813.811200:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299739965 00000020:00000001:3.0:1775767813.811201:0:9316:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767813.811205:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299738644 00000020:00000001:3.0:1775767813.811206:0:9316:0:(cl_io.c:528:cl_io_start()) Process entered 00020000:00000001:3.0:1775767813.811209:0:9316:0:(lov_io.c:1114:lov_io_start()) Process entered 00000100:00000001:2.0:1775767813.811210:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299739966 00020000:00000001:3.0:1775767813.811211:0:9316:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775767813.811213:0:9316:0:(cl_io.c:528:cl_io_start()) Process entered 00000100:00000001:2.0:1775767813.811215:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00000001:3.0:1775767813.811215:0:9316:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000008:00000001:3.0:1775767813.811217:0:9316:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000100:00000001:2.0:1775767813.811221:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1775767813.811222:0:9316:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:3.0:1775767813.811223:0:9316:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000100:00000001:2.0:1775767813.811226:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000008:00000001:3.0:1775767813.811226:0:9316:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1775767813.811229:0:9316:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000008:00000001:3.0:1775767813.811230:0:9316:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000008:00000001:3.0:1775767813.811232:0:9316:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107754938304 : -112965954613312 : ffff994210674fc0) 00000100:00100000:2.0:1775767813.811234:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767813/real 1775767813] req@ffff9942099fca80 x1862022294238208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000020:3.0:1775767813.811235:0:9316:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994210674fc0 ready 0|-|- wr 1|-|+ rd 0|- 9 in flight 00000008:00000001:3.0:1775767813.811240:0:9316:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:3.0:1775767813.811242:0:9316:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:3.0:1775767813.811243:0:9316:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1775767813.811246:0:9316:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000008:00000020:3.0:1775767813.811249:0:9316:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994218232380 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000008:00000001:3.0:1775767813.811256:0:9316:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:2.0:1775767813.811258:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000008:00000001:3.0:1775767813.811258:0:9316:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1775767813.811262:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:3.0:1775767813.811262:0:9316:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767813.811265:0:9316:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775767813.811266:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39c60. 00020000:00000001:3.0:1775767813.811267:0:9316:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767813.811270:0:9316:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767813.811272:0:9316:0:(cl_io.c:555:cl_io_end()) Process entered 00000400:00000200:2.0:1775767813.811273:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f220 00020000:00000001:3.0:1775767813.811273:0:9316:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00020000:00000001:3.0:1775767813.811275:0:9316:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000020:00000001:3.0:1775767813.811277:0:9316:0:(cl_io.c:555:cl_io_end()) Process entered 00000400:00000010:2.0:1775767813.811278:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f220. 00000020:00000001:3.0:1775767813.811278:0:9316:0:(cl_io.c:563:cl_io_end()) Process leaving 00020000:00000001:3.0:1775767813.811280:0:9316:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767813.811282:0:9316:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000100:00000001:2.0:1775767813.811283:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:3.0:1775767813.811283:0:9316:0:(cl_io.c:563:cl_io_end()) Process leaving 00000020:00000001:3.0:1775767813.811285:0:9316:0:(cl_io.c:352:cl_io_unlock()) Process entered 00020000:00000001:3.0:1775767813.811287:0:9316:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00020000:00000001:3.0:1775767813.811289:0:9316:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775767813.811290:0:9316:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000100:00000200:2.0:1775767813.811291:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fca80 x1862022294238208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775767813.811291:0:9316:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:3.0:1775767813.811293:0:9316:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767813.811297:0:9316:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767813.811300:0:9316:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000020:00000001:3.0:1775767813.811302:0:9316:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:3.0:1775767813.811305:0:9316:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000020:00000001:3.0:1775767813.811307:0:9316:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000001:3.0:1775767813.811309:0:9316:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:3.0:1775767813.811311:0:9316:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00020000:00000001:3.0:1775767813.811312:0:9316:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775767813.811314:0:9316:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000100:00000200:2.0:1775767813.811315:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fca80 x1862022294238208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775767813.811316:0:9316:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00020000:00000001:3.0:1775767813.811317:0:9316:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767813.811320:0:9316:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767813.811324:0:9316:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000020:00000001:3.0:1775767813.811325:0:9316:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000020:00000001:3.0:1775767813.811327:0:9316:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767813.811330:0:9316:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:3.0:1775767813.811332:0:9316:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:3.0:1775767813.811333:0:9316:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:3.0:1775767813.811336:0:9316:0:(cl_io.c:71:cl_io_fini()) Process entered 00000100:00000001:2.0:1775767813.811337:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:3.0:1775767813.811337:0:9316:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:3.0:1775767813.811338:0:9316:0:(cl_object.c:925:cl_env_put()) 1@ffff99420693ecf0 00000100:00000001:2.0:1775767813.811341:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:3.0:1775767813.811341:0:9316:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210dcf000. 00000100:00000001:2.0:1775767813.811346:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000010:3.0:1775767813.811346:0:9316:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207981180. 00000100:00000001:2.0:1775767813.811348:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00000010:3.0:1775767813.811349:0:9316:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994208b51650. 00020000:00000010:3.0:1775767813.811354:0:9316:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994218000000. 00000100:00100000:2.0:1775767813.811357:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fca80 x1862022294238208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00008000:00000010:3.0:1775767813.811357:0:9316:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994207e0ade8. 00000080:00000010:3.0:1775767813.811361:0:9316:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99420541b0a8. 00000080:00000010:3.0:1775767813.811365:0:9316:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077c1da8. 00000020:00000010:3.0:1775767813.811368:0:9316:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcf400. 00000008:00000010:3.0:1775767813.811371:0:9316:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942086f8160. 00020000:00000010:3.0:1775767813.811375:0:9316:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994207e8ed00. 00000100:00000001:2.0:1775767813.811378:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00008000:00000010:3.0:1775767813.811378:0:9316:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994209270ef0. 00000080:00000010:3.0:1775767813.811383:0:9316:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942090e3960. 00000100:00000040:2.0:1775767813.811387:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fca80 x1862022294238208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767838 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1775767813.811387:0:9316:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dce000. 00000020:00000010:3.0:1775767813.811389:0:9316:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff99420693ecf0. 00020000:00000001:3.0:1775767813.811392:0:9316:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:3.0:1775767813.811394:0:9316:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:3.0:1775767813.811395:0:9316:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000080:00200000:3.0:1775767813.811397:0:9316:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017f:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:3.0:1775767813.811403:0:9316:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:3.0:1775767813.811404:0:9316:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:3.0:1775767813.811406:0:9316:0:(cl_object.c:925:cl_env_put()) 1@ffff99420693e000 00000020:00000010:3.0:1775767813.811409:0:9316:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210dce200. 00000100:00000001:2.0:1775767813.811411:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000010:3.0:1775767813.811412:0:9316:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207981f00. 00000008:00000010:3.0:1775767813.811414:0:9316:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994208b50000. 00000100:00000001:2.0:1775767813.811418:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00020000:00000010:3.0:1775767813.811418:0:9316:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994218000300. 00008000:00000010:3.0:1775767813.811422:0:9316:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994207e08e20. 00000100:00000001:2.0:1775767813.811423:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000080:00000010:3.0:1775767813.811424:0:9316:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99420541b1f8. 00000100:00000001:2.0:1775767813.811426:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000080:00000010:3.0:1775767813.811427:0:9316:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077c0248. 00000020:00000010:3.0:1775767813.811431:0:9316:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcea00. 00000100:00000001:2.0:1775767813.811432:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000008:00000010:3.0:1775767813.811434:0:9316:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942086f8420. 00000100:00000001:2.0:1775767813.811436:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00020000:00000010:3.0:1775767813.811437:0:9316:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994207e8e400. 00008000:00000010:3.0:1775767813.811440:0:9316:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994209270a90. 00000100:00080000:2.0:1775767813.811443:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000080:00000010:3.0:1775767813.811443:0:9316:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942090e33c0. 00000020:00000010:3.0:1775767813.811446:0:9316:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dce600. 00000020:00000010:3.0:1775767813.811450:0:9316:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff99420693e000. 00000100:00080000:2.0:1775767813.811452:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000080:00000001:3.0:1775767813.811453:0:9316:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:3.0:1775767813.811456:0:9316:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767813.811458:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767813.811464:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fca80 x1862022294238208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767838 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767813.811474:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294238208:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775767813.811478:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767813.811482:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fe680 x1862022294238336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767813.811489:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767813.811491:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767813.811493:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767813.811495:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767813.811497:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767813.811500:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294238336:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775767813.811503:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767813.811505:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767813.811507:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767813.811509:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775767813.811512:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1fc00. 02000000:00000001:2.0:1775767813.811514:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767813.811518:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767813.811521:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39c60. 00000400:00000010:2.0:1775767813.811524:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f220. 00000100:00000200:2.0:1775767813.811527:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294238336, portal 10 00000100:00000001:2.0:1775767813.811529:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767813.811530:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641206400 : -112966068345216 : ffff9942099fe680) 00000100:00000040:2.0:1775767813.811536:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fe680 x1862022294238336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767838 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767813.811543:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767813.811546:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767813.811548:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775767813.811551:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294238336, offset 0 00000400:00000200:2.0:1775767813.811555:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767813.811560:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767813.811563:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767813.811566:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767813.811569:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767813.811572:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767813.811574:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767813.811577:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225035:225035:256:4294967295] 192.168.202.156@tcp LPNI seq info [225031:225031:8:4294967295] 00000400:00000200:2.0:1775767813.811582:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767813.811590:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767813.811595:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767813.811599:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3300. 00000800:00000200:2.0:1775767813.811603:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775767813.811606:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775767813.811610:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767813.811613:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000100:2.0:1775767813.811617:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767813.811619:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3300. 00000400:00000200:2.0:1775767813.811622:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775767813.811626:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775767813.811629:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775767813.811631:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775767813.811634:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767813.811637:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fe680 x1862022294238336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767838 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767813.811646:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767813.811649:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fe680 x1862022294238336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767838 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767813.811656:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767813.811658:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775767813.811661:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767813.811664:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:2.0:1775767813.811667:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767813.811669:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299738636 00000100:00000001:2.0:1775767813.811671:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299739965 00000100:00000001:2.0:1775767813.811673:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299738636 00000100:00000001:2.0:1775767813.811675:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299739966 00000100:00000001:2.0:1775767813.811677:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767813.811679:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767813.811681:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767813.811684:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767813/real 1775767813] req@ffff9942099fe680 x1862022294238336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775767813.811691:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767813.811693:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767813.811712:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39c60. 00000400:00000200:2.0:1775767813.811715:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f220 00000400:00000010:2.0:1775767813.811716:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f220. 00000100:00000001:2.0:1775767813.811719:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767813.811722:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fe680 x1862022294238336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767813.811731:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fe680 x1862022294238336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767813.811737:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767813.811739:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767813.811741:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767813.811742:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767813.811746:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fe680 x1862022294238336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767813.811753:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767813.811758:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fe680 x1862022294238336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767838 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767813.811765:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767813.811767:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767813.811768:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767813.811770:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767813.811771:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767813.811774:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767813.811776:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767813.811779:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767813.811781:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767813.811785:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fe680 x1862022294238336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767838 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767813.811792:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294238336:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775767813.811796:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767813.811799:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fdc00 x1862022294238464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767813.811805:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767813.811806:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767813.811808:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767813.811810:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767813.811812:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767813.811814:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294238464:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767813.811817:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767813.811819:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767813.811820:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767813.811822:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775767813.811825:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1e400. 02000000:00000001:2.0:1775767813.811827:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767813.811831:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767813.811833:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39c60. 00000400:00000010:2.0:1775767813.811836:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f220. 00000100:00000200:2.0:1775767813.811839:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294238464, portal 4 00000100:00000001:2.0:1775767813.811841:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767813.811842:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641203712 : -112966068347904 : ffff9942099fdc00) 00000100:00000040:2.0:1775767813.811847:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fdc00 x1862022294238464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767838 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767813.811855:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767813.811857:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767813.811859:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775767813.811861:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294238464, offset 0 00000400:00000200:2.0:1775767813.811865:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767813.811869:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767813.811873:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767813.811876:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767813.811878:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767813.811881:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767813.811883:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767813.811886:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225036:225036:256:4294967295] 192.168.202.156@tcp LPNI seq info [225032:225032:8:4294967295] 00000400:00000200:2.0:1775767813.811890:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767813.811896:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767813.811901:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767813.811904:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3300. 00000800:00000200:2.0:1775767813.811909:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775767813.811911:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775767813.811916:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767813.811918:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000100:2.0:1775767813.811922:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767813.811924:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3300. 00000400:00000200:2.0:1775767813.811927:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775767813.811930:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775767813.811934:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775767813.811936:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775767813.811938:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767813.811942:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fdc00 x1862022294238464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767838 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767813.811950:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767813.811954:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fdc00 x1862022294238464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767838 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767813.811961:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767813.811963:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775767813.811966:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767813.811969:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:2.0:1775767813.811973:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767813.811975:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299738659 00000100:00000001:2.0:1775767813.811977:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299739965 00000100:00000001:2.0:1775767813.811979:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299738659 00000100:00000001:2.0:1775767813.811982:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299739966 00000100:00000001:2.0:1775767813.811984:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767813.811987:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767813.811989:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767813.811993:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767813/real 1775767813] req@ffff9942099fdc00 x1862022294238464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775767813.812001:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767813.812003:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767813.812004:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39c60. 00000400:00000200:2.0:1775767813.812007:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f220 00000400:00000010:2.0:1775767813.812009:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f220. 00000100:00000001:2.0:1775767813.812011:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767813.812014:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fdc00 x1862022294238464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767813.812024:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fdc00 x1862022294238464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767813.812032:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767813.812034:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767813.812035:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767813.812037:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767813.812041:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fdc00 x1862022294238464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767813.812047:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767813.812051:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fdc00 x1862022294238464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767838 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767813.812057:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767813.812059:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767813.812061:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767813.812062:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767813.812064:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767813.812067:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767813.812070:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767813.812073:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767813.812075:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767813.812078:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fdc00 x1862022294238464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767838 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767813.812087:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294238464:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767813.812091:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767813.812094:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fce00 x1862022294238592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767813.812101:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767813.812102:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767813.812105:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767813.812106:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767813.812108:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767813.812111:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294238592:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767813.812115:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767813.812117:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767813.812118:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767813.812120:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775767813.812123:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f400. 02000000:00000001:2.0:1775767813.812125:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767813.812128:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767813.812131:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39c60. 00000400:00000010:2.0:1775767813.812134:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f220. 00000100:00000200:2.0:1775767813.812138:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294238592, portal 4 00000100:00000001:2.0:1775767813.812140:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767813.812141:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641200128 : -112966068351488 : ffff9942099fce00) 00000100:00000040:2.0:1775767813.812147:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fce00 x1862022294238592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767838 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767813.812172:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767813.812174:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767813.812177:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775767813.812179:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294238592, offset 0 00000400:00000200:2.0:1775767813.812181:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767813.812185:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767813.812187:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767813.812189:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767813.812191:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767813.812193:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767813.812194:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767813.812197:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225037:225037:256:4294967295] 192.168.202.156@tcp LPNI seq info [225033:225033:8:4294967295] 00000400:00000200:2.0:1775767813.812200:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767813.812205:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767813.812208:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767813.812210:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3300. 00000800:00000200:2.0:1775767813.812213:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775767813.812215:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775767813.812217:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767813.812219:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000100:2.0:1775767813.812222:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767813.812223:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3300. 00000400:00000200:2.0:1775767813.812225:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775767813.812227:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775767813.812229:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775767813.812231:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775767813.812232:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767813.812235:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fce00 x1862022294238592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767838 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767813.812240:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767813.812242:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fce00 x1862022294238592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767838 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767813.812246:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767813.812247:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775767813.812249:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767813.812252:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:2.0:1775767813.812253:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767813.812255:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299738663 00000100:00000001:2.0:1775767813.812256:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299739965 00000100:00000001:2.0:1775767813.812258:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299738663 00000100:00000001:2.0:1775767813.812259:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299739966 00000100:00000001:2.0:1775767813.812260:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767813.812262:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767813.812263:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767813.812265:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767813/real 1775767813] req@ffff9942099fce00 x1862022294238592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775767813.812271:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767813.812272:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767813.812273:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39c60. 00000400:00000200:2.0:1775767813.812274:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f220 00000400:00000010:2.0:1775767813.812275:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f220. 00000100:00000001:2.0:1775767813.812276:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767813.812278:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fce00 x1862022294238592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767813.812283:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fce00 x1862022294238592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767813.812288:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767813.812288:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767813.812290:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767813.812290:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767813.812292:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fce00 x1862022294238592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767813.812296:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767813.812298:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fce00 x1862022294238592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767838 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767813.812302:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767813.812304:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767813.812305:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767813.812306:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767813.812307:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767813.812309:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767813.812310:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767813.812312:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767813.812313:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767813.812316:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fce00 x1862022294238592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767838 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767813.812321:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294238592:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767813.812323:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767813.812324:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767813.812326:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fca80 x1862022294238208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767838 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767813.812330:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767813.812331:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767813.812332:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1d400. 02000000:00000001:2.0:1775767813.812334:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767813.812335:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767813.812336:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:2.0:1775767813.812337:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767813.812338:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002bc00. 02000000:00000001:2.0:1775767813.812340:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767813.812341:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767813.812341:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fca80. 00000100:00000001:2.0:1775767813.812343:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767813.812344:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767813.812345:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767813.812346:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fe680 x1862022294238336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767838 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767813.812351:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767813.812352:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767813.812352:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1fc00. 02000000:00000001:2.0:1775767813.812354:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767813.812354:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767813.812355:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:2.0:1775767813.812356:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767813.812357:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028000. 02000000:00000001:2.0:1775767813.812358:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767813.812359:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767813.812360:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fe680. 00000100:00000001:2.0:1775767813.812362:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767813.812363:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767813.812363:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767813.812365:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fdc00 x1862022294238464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767838 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767813.812369:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767813.812370:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767813.812371:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1e400. 02000000:00000001:2.0:1775767813.812372:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767813.812373:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767813.812374:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:2.0:1775767813.812375:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767813.812375:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029800. 02000000:00000001:2.0:1775767813.812377:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767813.812377:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767813.812378:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fdc00. 00000100:00000001:2.0:1775767813.812379:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767813.812380:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767813.812381:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767813.812382:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fce00 x1862022294238592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767838 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767813.812386:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767813.812387:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767813.812389:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f400. 02000000:00000001:2.0:1775767813.812391:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767813.812392:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767813.812393:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:2.0:1775767813.812395:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767813.812396:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029400. 02000000:00000001:2.0:1775767813.812398:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767813.812399:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767813.812400:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fce00. 00000100:00000001:2.0:1775767813.812402:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767813.812403:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767813.812405:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767813.812409:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767813.812411:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767813.812413:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775767813.812414:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767813.873328:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767813.873332:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767813.873335:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775767814.833231:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767814.833236:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767814.833237:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767814.833239:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767814.833242:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:2.0:1775767814.833244:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767814.833245:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767814.833249:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1775767814.833250:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775767814.833253:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767814.833254:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767814.833256:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767814.833258:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767814.833259:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767814.833264:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767814.833265:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767814.833268:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767814.833271:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767814.833343:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767814.833345:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767814.833346:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767814.833348:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767814.833350:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767814.833351:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767814.833352:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767814.833354:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767814.833355:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767815.153266:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299739966, at 4299739968 00000100:00000010:1.0:1775767815.153282:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d6120. 00000100:00000001:3.0:1775767815.857209:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767815.857215:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767815.857217:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767815.857220:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:3.0:1775767815.857222:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775767815.857227:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767815.857227:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767815.857231:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767815.857231:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767815.857232:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767815.857238:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:2.0:1775767815.857238:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767815.857239:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767815.857242:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767815.857243:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767815.857244:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1775767815.857249:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775767815.857249:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767815.857250:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767815.857254:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767815.857257:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767815.857258:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767815.857262:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767815.857263:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767815.857267:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767815.857269:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767815.857272:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767815.921425:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767815.921429:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767815.921432:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775767816.881244:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767816.881253:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767816.881255:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767816.881259:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767816.881264:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767816.881268:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767816.881270:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767816.881273:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767816.881275:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767816.881292:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767816.881292:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767816.881301:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767816.881301:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767816.881306:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767816.881307:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767816.881315:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:1.0:1775767816.881316:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775767816.881323:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767816.881325:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767816.881330:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767816.881333:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767816.881336:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767816.881337:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767816.881341:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767816.881342:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767816.881350:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767816.881355:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767817.905247:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767817.905248:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767817.905255:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767817.905255:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767817.905259:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767817.905259:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767817.905269:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775767817.905269:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767817.905276:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767817.905277:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767817.905280:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767817.905281:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767817.905287:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767817.905289:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767817.905292:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767817.905293:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767817.905294:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767817.905297:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767817.905298:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767817.905302:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767817.905313:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767817.905317:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767817.905320:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767817.905322:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767817.905325:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767817.905326:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767817.905328:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767817.969391:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767817.969395:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767817.969399:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775767818.929237:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:2.0:1775767818.929237:0:9316:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1775767818.929237:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767818.929245:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767818.929245:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:2.0:1775767818.929247:0:9316:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775767818.929248:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767818.929249:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767818.929251:0:9316:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:3.0:1775767818.929254:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:1.0:1775767818.929256:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767818.929257:0:9316:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767818.929261:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767818.929263:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767818.929265:0:9316:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775767818.929267:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767818.929267:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1775767818.929274:0:9316:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5071 00000100:00000001:3.0:1775767818.929275:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767818.929280:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767818.929284:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767818.929285:0:9316:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767818.929286:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767818.929290:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767818.929293:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767818.929294:0:9316:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775767818.929298:0:9316:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775767818.929302:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767818.929306:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767818.929311:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767818.929315:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775767818.929320:0:9316:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775767818.929323:0:9316:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:2.0:1775767818.929326:0:9316:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775767818.929329:0:9316:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:2.0:1775767818.929334:0:9316:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:2.0:1775767818.929340:0:9316:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:2.0:1775767818.929343:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767818.929345:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767818.929347:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767818.929349:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775767818.929352:0:9316:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:2.0:1775767818.929354:0:9316:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775767818.929356:0:9316:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767818.929359:0:9316:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775767818.929361:0:9316:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767818.929363:0:9316:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767818.929365:0:9316:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775767818.929367:0:9316:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775767818.929375:0:9316:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfdc00. 00000020:00000040:2.0:1775767818.929378:0:9316:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:2.0:1775767818.929381:0:9316:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775767818.929382:0:9316:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767818.929385:0:9316:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775767818.929388:0:9316:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775767818.929392:0:9316:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775767818.929400:0:9316:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1f400. 00000100:00000001:2.0:1775767818.929406:0:9316:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767818.929415:0:9316:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfdc00 x1862022294238720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775767818.929424:0:9316:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775767818.929429:0:9316:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767818.929432:0:9316:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003e25603c] to pc [ptlrpcd_rcv+-1] req@ffff994207dfdc00 x1862022294238720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767818.929450:0:9316:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775767818.929453:0:9316:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767818.929456:0:9316:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775767818.929457:0:9316:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775767818.929458:0:9316:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775767818.929460:0:9316:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775767818.929462:0:9316:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5071 00000400:00000200:2.0:1775767818.929465:0:9316:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767818.929468:0:9316:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775767818.929469:0:9316:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775767818.929472:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767818.929473:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767818.929475:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767818.929476:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775767818.929477:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775767818.929480:0:9316:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775767818.929481:0:9316:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000100:00000001:3.0:1775767818.929482:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1775767818.929483:0:9316:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775767818.929486:0:9316:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:2.0:1775767818.929487:0:9316:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:2.0:1775767818.929489:0:9316:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:2.0:1775767818.929491:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767818.929492:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767818.929493:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767818.929495:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:3.0:1775767818.929496:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000040:2.0:1775767818.929497:0:9316:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00080000:2.0:1775767818.929499:0:9316:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775767818.929503:0:9316:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767818.929505:0:9316:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775767818.929510:0:9316:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1775767818.929510:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfdc00 x1862022294238720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1775767818.929513:0:9316:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767818.929516:0:9316:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775767818.929522:0:9316:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1775767818.929525:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000008:00000001:2.0:1775767818.929526:0:9316:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767818.929530:0:9316:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767818.929530:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775767818.929533:0:9316:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dff100. 00000020:00000040:2.0:1775767818.929535:0:9316:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:2.0:1775767818.929537:0:9316:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767818.929537:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767818.929540:0:9316:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767818.929541:0:9316:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767818.929542:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767818.929543:0:9316:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775767818.929545:0:9316:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775767818.929549:0:9316:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1e400. 02000000:00000001:3.0:1775767818.929549:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767818.929555:0:9316:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767818.929558:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294238720:192.168.202.156@tcp:250:kworker.0 00000100:00100000:2.0:1775767818.929562:0:9316:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dff100 x1862022294238848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767818.929567:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1775767818.929572:0:9316:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775767818.929573:0:9316:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767818.929574:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:2.0:1775767818.929577:0:9316:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3e64ae1] to pc [ptlrpcd_rcv+-1] req@ffff994207dff100 x1862022294238848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1775767818.929579:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767818.929585:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1775767818.929589:0:9316:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767818.929593:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b2c00. 02000000:00000001:3.0:1775767818.929597:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775767818.929601:0:9316:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767818.929604:0:9316:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775767818.929605:0:9316:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775767818.929607:0:9316:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:3.0:1775767818.929609:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767818.929610:0:9316:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775767818.929614:0:9316:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5071 00000400:00000010:3.0:1775767818.929616:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000200:2.0:1775767818.929617:0:9316:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767818.929623:0:9316:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775767818.929625:0:9316:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775767818.929627:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767818.929628:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:3.0:1775767818.929629:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000040:2.0:1775767818.929630:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767818.929635:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775767818.929636:0:9316:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775767818.929637:0:9316:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:2.0:1775767818.929638:0:9316:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775767818.929639:0:9316:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:2.0:1775767818.929640:0:9316:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000200:3.0:1775767818.929642:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294238720, portal 25 00000020:00000001:2.0:1775767818.929643:0:9316:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000001:3.0:1775767818.929660:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000040:2.0:1775767818.929663:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767818.929664:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611843584 : -112966097708032 : ffff994207dfdc00) 00000100:00000001:2.0:1775767818.929666:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767818.929669:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767818.929671:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775767818.929675:0:9316:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00000040:3.0:1775767818.929676:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfdc00 x1862022294238720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767843 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767818.929682:0:9316:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775767818.929689:0:9316:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767818.929694:0:9316:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1775767818.929696:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:2.0:1775767818.929698:0:9316:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767818.929704:0:9316:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767818.929704:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000100:00000001:2.0:1775767818.929709:0:9316:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000010:3.0:1775767818.929711:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000008:00100000:2.0:1775767818.929712:0:9316:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775767818.929717:0:9316:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1775767818.929718:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294238720, offset 0 00000100:00000001:2.0:1775767818.929721:0:9316:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775767818.929727:0:9316:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfc700. 00000400:00000200:3.0:1775767818.929729:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000020:00000040:2.0:1775767818.929730:0:9316:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:2.0:1775767818.929734:0:9316:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775767818.929735:0:9316:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767818.929736:0:9316:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775767818.929737:0:9316:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775767818.929739:0:9316:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767818.929743:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000010:2.0:1775767818.929745:0:9316:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1fc00. 00000100:00000001:2.0:1775767818.929749:0:9316:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767818.929752:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00100000:2.0:1775767818.929756:0:9316:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfc700 x1862022294238976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:3.0:1775767818.929761:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1775767818.929766:0:9316:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775767818.929768:0:9316:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767818.929770:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000100:00000040:2.0:1775767818.929774:0:9316:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a176dd1] to pc [ptlrpcd_rcv+-1] req@ffff994207dfc700 x1862022294238976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1775767818.929780:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000100:00000001:2.0:1775767818.929785:0:9316:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767818.929787:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00080000:2.0:1775767818.929791:0:9316:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:3.0:1775767818.929796:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225038:225038:256:4294967295] 192.168.202.156@tcp LPNI seq info [225034:225034:8:4294967295] 00000100:00080000:2.0:1775767818.929799:0:9316:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1775767818.929802:0:9316:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775767818.929806:0:9316:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:3.0:1775767818.929806:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767818.929812:0:9316:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775767818.929817:0:9316:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5071 00000400:00000200:2.0:1775767818.929826:0:9316:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767818.929827:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000400:00000200:2.0:1775767818.929835:0:9316:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775767818.929841:0:9316:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000800:00000200:3.0:1775767818.929841:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00000040:2.0:1775767818.929848:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000800:00000010:3.0:1775767818.929849:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000100:00000001:2.0:1775767818.929854:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767818.929858:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000800:00000200:3.0:1775767818.929863:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:2.0:1775767818.929864:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775767818.929869:0:9316:0:(genops.c:811:class_conn2export()) Process entered 00000800:00000200:3.0:1775767818.929870:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000020:00000040:2.0:1775767818.929871:0:9316:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:2.0:1775767818.929874:0:9316:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775767818.929878:0:9316:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:2.0:1775767818.929883:0:9316:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000800:00000200:3.1:1775767818.929883:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000001:2.0:1775767818.929887:0:9316:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000800:00000200:3.1:1775767818.929890:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000100:00000040:2.0:1775767818.929894:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767818.929898:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000100:3.0:1775767818.929902:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000100:00000040:2.0:1775767818.929905:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000800:00000010:3.0:1775767818.929907:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000100:00000001:2.0:1775767818.929909:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775767818.929913:0:9316:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000400:00000200:3.0:1775767818.929914:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00080000:2.0:1775767818.929918:0:9316:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775767818.929922:0:9316:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767818.929925:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 02000000:00000001:2.0:1775767818.929926:0:9316:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775767818.929931:0:9316:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1775767818.929932:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 02000000:00000001:2.0:1775767818.929934:0:9316:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1775767818.929936:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:2.0:1775767818.929939:0:9316:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775767818.929942:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000008:00100000:2.0:1775767818.929943:0:9316:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:2.0:1775767818.929949:0:9316:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1775767818.929949:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfdc00 x1862022294238720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767843 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767818.929952:0:9316:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775767818.929958:0:9316:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfe300. 00000020:00000040:2.0:1775767818.929961:0:9316:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:2.0:1775767818.929965:0:9316:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775767818.929967:0:9316:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767818.929969:0:9316:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1775767818.929969:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1775767818.929972:0:9316:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775767818.929975:0:9316:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767818.929976:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfdc00 x1862022294238720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767843 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1775767818.929981:0:9316:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1d400. 00000100:00000001:2.0:1775767818.929988:0:9316:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767818.929992:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767818.929997:0:9316:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfe300 x1862022294239104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775767818.930006:0:9316:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775767818.930008:0:9316:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767818.930010:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:2.0:1775767818.930015:0:9316:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c661fef] to pc [ptlrpcd_rcv+-1] req@ffff994207dfe300 x1862022294239104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767818.930019:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767818.930027:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767818.930031:0:9316:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767818.930031:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299745085 00000100:00080000:2.0:1775767818.930035:0:9316:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000010:3.0:1775767818.930038:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207981f00. 00000100:00080000:2.0:1775767818.930039:0:9316:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1775767818.930042:0:9316:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775767818.930043:0:9316:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775767818.930046:0:9316:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:3.0:1775767818.930048:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00080000:2.0:1775767818.930049:0:9316:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5071 00000400:00000200:2.0:1775767818.930055:0:9316:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767818.930058:0:9316:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775767818.930060:0:9316:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1775767818.930060:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299745085 00000100:00000040:2.0:1775767818.930065:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000010:3.0:1775767818.930067:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207981180. 00000100:00000001:2.0:1775767818.930068:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767818.930071:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767818.930073:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1775767818.930074:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775767818.930078:0:9316:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775767818.930080:0:9316:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000100:00000001:3.0:1775767818.930080:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1775767818.930081:0:9316:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775767818.930084:0:9316:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:2.0:1775767818.930087:0:9316:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000001:3.0:1775767818.930087:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767818.930091:0:9316:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000001:3.0:1775767818.930093:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000040:2.0:1775767818.930094:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767818.930097:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767818.930098:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767818.930101:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775767818.930103:0:9316:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00100000:3.0:1775767818.930103:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767818/real 1775767818] req@ffff994207dfdc00 x1862022294238720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767818.930110:0:9316:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775767818.930115:0:9316:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767818.930118:0:9316:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000001:3.0:1775767818.930123:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:2.0:1775767818.930124:0:9316:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000001:3.0:1775767818.930126:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:2.0:1775767818.930128:0:9316:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1775767818.930129:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000100:00000001:2.0:1775767818.930133:0:9316:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:3.0:1775767818.930133:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767818.930136:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000008:00100000:2.0:1775767818.930137:0:9316:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000100:00000001:3.0:1775767818.930139:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000008:00000001:2.0:1775767818.930142:0:9316:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1775767818.930145:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfdc00 x1862022294238720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767818.930146:0:9316:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775767818.930153:0:9316:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfed80. 00000020:00000040:2.0:1775767818.930183:0:9316:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00000100:00000200:3.0:1775767818.930185:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfdc00 x1862022294238720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1775767818.930189:0:9316:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775767818.930192:0:9316:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767818.930193:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767818.930195:0:9316:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1775767818.930196:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1775767818.930199:0:9316:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1775767818.930199:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:2.0:1775767818.930203:0:9316:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767818.930204:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767818.930209:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfdc00 x1862022294238720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1775767818.930211:0:9316:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1c800. 00000100:00000001:2.0:1775767818.930219:0:9316:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767818.930219:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767818.930231:0:9316:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfed80 x1862022294239232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:3.0:1775767818.930231:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfdc00 x1862022294238720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767843 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767818.930286:0:9316:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767818.930286:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767818.930291:0:9316:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767818.930292:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767818.930296:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767818.930298:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767818.930301:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000040:2.0:1775767818.930302:0:9316:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb9b9eae] to pc [ptlrpcd_rcv+-1] req@ffff994207dfed80 x1862022294239232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767818.930305:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767818.930309:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767818.930316:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767818.930320:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1775767818.930322:0:9316:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767818.930325:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfdc00 x1862022294238720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767843 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775767818.930329:0:9316:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5081) 00000100:00100000:3.0:1775767818.930336:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294238720:192.168.202.156@tcp:250:kworker.0 00000100:00000001:3.0:1775767818.930343:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767818.930345:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767818.930350:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfdc00 x1862022294238720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767843 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767818.930358:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767818.930361:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767818.930365:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b2c00. 02000000:00000001:3.0:1775767818.930368:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767818.930370:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767818.930371:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:3.0:1775767818.930374:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767818.930376:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1f400. 02000000:00000001:3.0:1775767818.930380:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767818.930382:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767818.930385:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfdc00. 00000100:00000001:3.0:1775767818.930388:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767818.930390:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767818.930392:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767818.930401:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767818.930403:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767818.930405:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767818.930407:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767818.930408:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767818.930413:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dff100 x1862022294238848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767818.930420:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767818.930422:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767818.930424:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767818.930425:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767818.930428:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767818.930430:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294238848:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767818.930434:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767818.930436:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767818.930437:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767818.930439:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767818.930442:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b2c00. 02000000:00000001:3.0:1775767818.930444:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767818.930449:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767818.930451:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767818.930455:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767818.930458:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294238848, portal 10 00000100:00000001:3.0:1775767818.930460:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767818.930461:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611848960 : -112966097702656 : ffff994207dff100) 00000100:00000040:3.0:1775767818.930468:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dff100 x1862022294238848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767843 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767818.930475:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767818.930477:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767818.930480:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767818.930482:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294238848, offset 0 00000400:00000200:3.0:1775767818.930486:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767818.930491:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767818.930496:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767818.930500:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767818.930502:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767818.930505:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767818.930507:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767818.930509:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225039:225039:256:4294967295] 192.168.202.156@tcp LPNI seq info [225035:225035:8:4294967295] 00000400:00000200:3.0:1775767818.930513:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767818.930520:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767818.930526:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767818.930530:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767818.930534:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767818.930537:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775767818.930542:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767818.930545:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1775767818.930549:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767818.930551:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767818.930554:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767818.930558:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767818.930561:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767818.930563:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767818.930566:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767818.930570:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dff100 x1862022294238848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767843 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767818.930579:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767818.930583:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dff100 x1862022294238848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767843 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767818.930590:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767818.930591:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767818.930595:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767818.930599:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:3.0:1775767818.930603:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767818.930605:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299743763 00000100:00000001:3.0:1775767818.930609:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299745085 00000100:00000001:3.0:1775767818.930612:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299743763 00000100:00000001:3.0:1775767818.930614:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299745085 00000100:00000001:3.0:1775767818.930616:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767818.930619:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767818.930621:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767818.930624:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767818/real 1775767818] req@ffff994207dff100 x1862022294238848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767818.930633:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767818.930634:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767818.930636:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767818.930638:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767818.930640:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767818.930642:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767818.930656:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dff100 x1862022294238848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767818.930667:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dff100 x1862022294238848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767818.930674:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767818.930676:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767818.930678:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767818.930680:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767818.930684:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dff100 x1862022294238848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767818.930692:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767818.930696:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dff100 x1862022294238848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767843 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767818.930704:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767818.930706:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767818.930709:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767818.930710:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767818.930712:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767818.930715:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767818.930718:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767818.930721:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767818.930723:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767818.930727:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dff100 x1862022294238848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767843 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767818.930736:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294238848:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767818.930740:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767818.930743:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfc700 x1862022294238976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767818.930750:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767818.930752:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767818.930754:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767818.930756:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767818.930758:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767818.930761:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294238976:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767818.930765:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767818.930767:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767818.930769:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767818.930771:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767818.930775:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b0c00. 02000000:00000001:3.0:1775767818.930778:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767818.930782:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767818.930784:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767818.930788:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767818.930791:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294238976, portal 10 00000100:00000001:3.0:1775767818.930793:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767818.930796:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611838208 : -112966097713408 : ffff994207dfc700) 00000100:00000040:3.0:1775767818.930801:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfc700 x1862022294238976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767843 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767818.930808:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767818.930811:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767818.930813:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767818.930816:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294238976, offset 0 00000400:00000200:3.0:1775767818.930820:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767818.930824:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767818.930828:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767818.930831:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767818.930834:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767818.930837:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767818.930839:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767818.930842:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225040:225040:256:4294967295] 192.168.202.156@tcp LPNI seq info [225036:225036:8:4294967295] 00000400:00000200:3.0:1775767818.930847:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767818.930854:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767818.930860:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767818.930863:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767818.930868:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767818.930871:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775767818.930875:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767818.930878:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1775767818.930882:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767818.930884:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767818.930887:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767818.930891:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767818.930894:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767818.930897:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767818.930899:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767818.930903:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfc700 x1862022294238976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767843 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767818.930919:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767818.930922:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfc700 x1862022294238976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767843 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767818.930927:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767818.930928:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767818.930931:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767818.930933:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:3.0:1775767818.930936:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767818.930937:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299743755 00000100:00000001:3.0:1775767818.930939:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299745085 00000100:00000001:3.0:1775767818.930941:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299743755 00000100:00000001:3.0:1775767818.930943:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299745085 00000100:00000001:3.0:1775767818.930944:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767818.930946:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767818.930947:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767818.930950:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767818/real 1775767818] req@ffff994207dfc700 x1862022294238976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767818.930955:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767818.930955:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767818.930956:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767818.930958:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767818.930959:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767818.930960:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767818.930962:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfc700 x1862022294238976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767818.930969:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfc700 x1862022294238976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767818.930977:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767818.930979:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767818.930981:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767818.930982:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767818.930986:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfc700 x1862022294238976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767818.930993:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767818.930997:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfc700 x1862022294238976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767843 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767818.931005:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767818.931007:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767818.931009:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767818.931010:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767818.931012:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767818.931015:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767818.931018:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767818.931022:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767818.931024:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767818.931028:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfc700 x1862022294238976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767843 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767818.931037:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294238976:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767818.931041:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767818.931044:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfe300 x1862022294239104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767818.931051:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767818.931052:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767818.931054:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767818.931056:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767818.931058:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767818.931062:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294239104:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767818.931065:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767818.931067:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767818.931069:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767818.931071:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767818.931075:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b2800. 02000000:00000001:3.0:1775767818.931077:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767818.931080:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767818.931082:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767818.931085:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767818.931087:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294239104, portal 4 00000100:00000001:3.0:1775767818.931088:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767818.931089:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611845376 : -112966097706240 : ffff994207dfe300) 00000100:00000040:3.0:1775767818.931095:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfe300 x1862022294239104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767843 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767818.931102:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767818.931105:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767818.931107:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767818.931110:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294239104, offset 0 00000400:00000200:3.0:1775767818.931115:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767818.931119:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767818.931123:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767818.931126:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767818.931128:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767818.931131:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767818.931133:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767818.931137:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225041:225041:256:4294967295] 192.168.202.156@tcp LPNI seq info [225037:225037:8:4294967295] 00000400:00000200:3.0:1775767818.931142:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767818.931148:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767818.931177:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767818.931182:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767818.931188:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767818.931191:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775767818.931194:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767818.931196:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1775767818.931200:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767818.931203:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767818.931205:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767818.931208:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767818.931211:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767818.931212:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767818.931215:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767818.931219:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfe300 x1862022294239104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767843 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767818.931228:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767818.931233:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfe300 x1862022294239104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767843 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767818.931239:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767818.931241:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767818.931245:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767818.931249:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:3.0:1775767818.931252:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767818.931254:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299743779 00000100:00000001:3.0:1775767818.931257:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299745085 00000100:00000001:3.0:1775767818.931259:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299743779 00000100:00000001:3.0:1775767818.931261:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299745085 00000100:00000001:3.0:1775767818.931263:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767818.931265:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767818.931267:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767818.931271:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767818/real 1775767818] req@ffff994207dfe300 x1862022294239104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767818.931278:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767818.931280:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767818.931282:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767818.931284:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767818.931286:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767818.931288:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767818.931292:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfe300 x1862022294239104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767818.931300:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfe300 x1862022294239104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767818.931304:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767818.931305:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767818.931307:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767818.931308:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767818.931310:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfe300 x1862022294239104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767818.931314:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767818.931317:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfe300 x1862022294239104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767843 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767818.931321:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767818.931323:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767818.931324:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767818.931324:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767818.931326:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767818.931329:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767818.931330:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767818.931333:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767818.931334:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767818.931336:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfe300 x1862022294239104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767843 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767818.931345:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294239104:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767818.931348:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767818.931350:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfed80 x1862022294239232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767818.931354:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767818.931355:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767818.931356:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767818.931357:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767818.931359:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767818.931361:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294239232:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767818.931364:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767818.931366:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767818.931367:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767818.931369:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767818.931373:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b0000. 02000000:00000001:3.0:1775767818.931375:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767818.931379:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767818.931382:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767818.931385:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767818.931388:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294239232, portal 4 00000100:00000001:3.0:1775767818.931390:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767818.931391:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611848064 : -112966097703552 : ffff994207dfed80) 00000100:00000040:3.0:1775767818.931397:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfed80 x1862022294239232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767843 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767818.931404:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767818.931407:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767818.931409:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767818.931412:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294239232, offset 0 00000400:00000200:3.0:1775767818.931416:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767818.931420:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767818.931424:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767818.931427:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767818.931430:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767818.931432:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767818.931435:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767818.931438:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225042:225042:256:4294967295] 192.168.202.156@tcp LPNI seq info [225038:225038:8:4294967295] 00000400:00000200:3.0:1775767818.931443:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767818.931449:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767818.931454:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767818.931458:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767818.931462:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767818.931465:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775767818.931469:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767818.931472:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1775767818.931475:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767818.931477:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767818.931480:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767818.931484:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767818.931487:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767818.931489:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767818.931491:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767818.931495:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfed80 x1862022294239232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767843 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767818.931503:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767818.931506:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfed80 x1862022294239232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767843 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767818.931511:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767818.931513:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767818.931515:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767818.931518:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:3.0:1775767818.931520:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767818.931522:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299743782 00000100:00000001:3.0:1775767818.931524:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299745085 00000100:00000001:3.0:1775767818.931526:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299743782 00000100:00000001:3.0:1775767818.931527:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299745085 00000100:00000001:3.0:1775767818.931529:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767818.931530:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767818.931532:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767818.931534:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767818/real 1775767818] req@ffff994207dfed80 x1862022294239232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767818.931539:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767818.931539:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767818.931540:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767818.931542:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767818.931543:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767818.931544:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767818.931546:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfed80 x1862022294239232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767818.931551:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfed80 x1862022294239232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767818.931557:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767818.931559:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767818.931561:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767818.931562:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767818.931566:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfed80 x1862022294239232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767818.931573:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767818.931576:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfed80 x1862022294239232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767843 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767818.931585:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767818.931588:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767818.931589:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767818.931590:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767818.931591:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767818.931594:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767818.931596:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767818.931599:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767818.931600:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767818.931603:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfed80 x1862022294239232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767843 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767818.931609:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294239232:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767818.931632:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767818.931633:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767818.931635:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dff100 x1862022294238848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767843 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767818.931640:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767818.931641:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767818.931643:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b2c00. 02000000:00000001:3.0:1775767818.931673:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767818.931675:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767818.931677:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:3.0:1775767818.931680:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767818.931682:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1e400. 02000000:00000001:3.0:1775767818.931690:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767818.931691:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767818.931693:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dff100. 00000100:00000001:3.0:1775767818.931696:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767818.931697:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767818.931699:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767818.931703:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfc700 x1862022294238976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767843 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767818.931717:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767818.931718:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767818.931720:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b0c00. 02000000:00000001:3.0:1775767818.931723:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767818.931724:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767818.931725:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:3.0:1775767818.931727:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767818.931729:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1fc00. 02000000:00000001:3.0:1775767818.931730:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767818.931732:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767818.931733:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfc700. 00000100:00000001:3.0:1775767818.931735:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767818.931737:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767818.931739:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767818.931742:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfe300 x1862022294239104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767843 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767818.931750:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767818.931752:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767818.931754:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b2800. 02000000:00000001:3.0:1775767818.931757:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767818.931759:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767818.931760:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:3.0:1775767818.931762:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767818.931764:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1d400. 02000000:00000001:3.0:1775767818.931767:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767818.931768:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767818.931769:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfe300. 00000100:00000001:3.0:1775767818.931771:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767818.931772:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767818.931773:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767818.931775:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfed80 x1862022294239232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767843 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767818.931780:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767818.931781:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767818.931782:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b0000. 02000000:00000001:3.0:1775767818.931784:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767818.931785:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767818.931786:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:3.0:1775767818.931787:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767818.931788:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1c800. 02000000:00000001:3.0:1775767818.931789:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767818.931790:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767818.931791:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfed80. 00000100:00000001:3.0:1775767818.931792:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767818.931793:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767818.931794:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767818.931797:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767818.931798:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767818.931799:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767818.931800:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767819.953250:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767819.953255:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767819.953259:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767819.953265:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767819.953272:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767819.953277:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767819.953284:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767819.953288:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767819.953291:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767819.953394:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767819.953397:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767819.953399:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767819.953402:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767819.953405:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767819.953407:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767819.953410:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767819.953412:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767819.953414:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767820.017449:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767820.017454:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767820.017458:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775767820.273233:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299745085, at 4299745088 00000100:00000010:1.0:1775767820.273249:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207981180. 00000100:00000001:1.0:1775767820.977241:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767820.977248:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767820.977251:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767820.977257:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767820.977263:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767820.977268:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767820.977274:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767820.977281:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767820.977282:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767820.977288:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767820.977289:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767820.977292:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767820.977298:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775767820.977303:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767820.977306:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767820.977308:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767820.977310:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767820.977313:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767822.001236:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767822.001243:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767822.001246:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767822.001252:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775767822.001258:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767822.001264:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767822.001265:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767822.001267:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767822.001275:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767822.001275:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767822.001281:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767822.001288:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775767822.001290:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767822.001293:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767822.001299:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767822.001309:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767822.001312:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767822.001315:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767822.065418:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767822.065422:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767822.065425:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775767823.025273:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767823.025281:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767823.025285:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767823.025293:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775767823.025299:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767823.025305:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767823.025316:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767823.025319:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767823.025321:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767823.025400:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767823.025404:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767823.025405:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767823.025408:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767823.025410:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767823.025411:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767823.025413:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767823.025414:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767823.025416:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767824.049310:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767824.049319:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:2.0:1775767824.049322:0:9316:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775767824.049323:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767824.049330:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:1.0:1775767824.049332:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00080000:2.0:1775767824.049333:0:9316:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775767824.049336:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767824.049339:0:9316:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1775767824.049340:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767824.049341:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767824.049346:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767824.049346:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00080000:2.0:1775767824.049347:0:9316:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767824.049350:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767824.049355:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767824.049356:0:9316:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775767824.049362:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767824.049366:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1775767824.049368:0:9316:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5076 00000100:00000001:1.0:1775767824.049371:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767824.049371:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767824.049375:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767824.049378:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767824.049383:0:9316:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767824.049394:0:9316:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775767824.049399:0:9316:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775767824.049404:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767824.049408:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767824.049412:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767824.049416:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775767824.049422:0:9316:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775767824.049424:0:9316:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:2.0:1775767824.049428:0:9316:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775767824.049431:0:9316:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:2.0:1775767824.049434:0:9316:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:2.0:1775767824.049437:0:9316:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:2.0:1775767824.049441:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767824.049443:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767824.049446:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767824.049448:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775767824.049451:0:9316:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:2.0:1775767824.049455:0:9316:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775767824.049458:0:9316:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767824.049463:0:9316:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775767824.049467:0:9316:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767824.049470:0:9316:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767824.049473:0:9316:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775767824.049477:0:9316:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775767824.049490:0:9316:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dff480. 00000020:00000040:2.0:1775767824.049505:0:9316:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:2.0:1775767824.049510:0:9316:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775767824.049513:0:9316:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767824.049516:0:9316:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775767824.049520:0:9316:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775767824.049526:0:9316:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775767824.049536:0:9316:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1ec00. 00000100:00000001:2.0:1775767824.049544:0:9316:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767824.049557:0:9316:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dff480 x1862022294239360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775767824.049568:0:9316:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775767824.049574:0:9316:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767824.049580:0:9316:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093a15639] to pc [ptlrpcd_rcv+-1] req@ffff994207dff480 x1862022294239360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767824.049614:0:9316:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775767824.049618:0:9316:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767824.049622:0:9316:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775767824.049623:0:9316:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775767824.049625:0:9316:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775767824.049629:0:9316:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775767824.049632:0:9316:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5076 00000400:00000200:2.0:1775767824.049638:0:9316:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767824.049642:0:9316:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775767824.049644:0:9316:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1775767824.049645:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1775767824.049650:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767824.049652:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767824.049654:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1775767824.049656:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767824.049660:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767824.049664:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000040:3.0:1775767824.049668:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dff480 x1862022294239360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1775767824.049669:0:9316:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775767824.049672:0:9316:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:2.0:1775767824.049676:0:9316:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775767824.049679:0:9316:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:2.0:1775767824.049685:0:9316:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:2.0:1775767824.049691:0:9316:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000001:3.0:1775767824.049691:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767824.049695:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767824.049699:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 02000000:00000001:3.0:1775767824.049702:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1775767824.049703:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767824.049707:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000001:3.0:1775767824.049708:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767824.049712:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 02000000:00000001:3.0:1775767824.049713:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775767824.049718:0:9316:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00100000:3.0:1775767824.049720:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294239360:192.168.202.156@tcp:250:kworker.0 00000100:00080000:2.0:1775767824.049726:0:9316:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767824.049731:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1775767824.049732:0:9316:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767824.049735:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767824.049736:0:9316:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767824.049739:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767824.049743:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1775767824.049745:0:9316:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000010:3.0:1775767824.049748:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b0000. 02000000:00000001:2.0:1775767824.049749:0:9316:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767824.049751:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767824.049754:0:9316:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:3.0:1775767824.049759:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00100000:2.0:1775767824.049760:0:9316:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000400:00000010:3.0:1775767824.049763:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000008:00000001:2.0:1775767824.049765:0:9316:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767824.049769:0:9316:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1775767824.049770:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000010:2.0:1775767824.049774:0:9316:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfce00. 00000020:00000040:2.0:1775767824.049778:0:9316:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000100:00000200:3.0:1775767824.049779:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294239360, portal 25 02000000:00000001:2.0:1775767824.049783:0:9316:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1775767824.049784:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1775767824.049788:0:9316:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767824.049788:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611849856 : -112966097701760 : ffff994207dff480) 00000100:00000001:2.0:1775767824.049794:0:9316:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775767824.049798:0:9316:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:3.0:1775767824.049798:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dff480 x1862022294239360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767849 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1775767824.049802:0:9316:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767824.049810:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000010:2.0:1775767824.049812:0:9316:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1e800. 00000100:00000040:3.0:1775767824.049814:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767824.049819:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000001:2.0:1775767824.049820:0:9316:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1775767824.049824:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294239360, offset 0 00000100:00100000:2.0:1775767824.049830:0:9316:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfce00 x1862022294239488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:3.0:1775767824.049834:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00000001:2.0:1775767824.049844:0:9316:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:3.0:1775767824.049846:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767824.049851:0:9316:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767824.049854:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00000040:2.0:1775767824.049860:0:9316:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e5a7debe] to pc [ptlrpcd_rcv+-1] req@ffff994207dfce00 x1862022294239488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1775767824.049861:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767824.049867:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767824.049874:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767824.049879:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00000001:2.0:1775767824.049880:0:9316:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767824.049885:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225043:225043:256:4294967295] 192.168.202.156@tcp LPNI seq info [225039:225039:8:4294967295] 00000100:00080000:2.0:1775767824.049887:0:9316:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:3.0:1775767824.049893:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00080000:2.0:1775767824.049897:0:9316:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775767824.049899:0:9316:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775767824.049902:0:9316:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775767824.049910:0:9316:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:3.0:1775767824.049910:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000100:00080000:2.0:1775767824.049918:0:9316:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5076 00000800:00000200:3.0:1775767824.049919:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767824.049926:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000400:00000200:2.0:1775767824.049929:0:9316:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000800:00000200:3.0:1775767824.049937:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000400:00000200:2.0:1775767824.049938:0:9316:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000800:00000200:3.0:1775767824.049943:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000100:00080000:2.0:1775767824.049944:0:9316:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775767824.049953:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000800:00000200:3.1:1775767824.049954:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:2.0:1775767824.049960:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:3.1:1775767824.049960:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000100:00000040:2.0:1775767824.049967:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000800:00000100:3.0:1775767824.049968:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767824.049972:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000100:00000001:2.0:1775767824.049973:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775767824.049977:0:9316:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775767824.049978:0:9316:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000400:00000200:3.0:1775767824.049980:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000001:2.0:1775767824.049981:0:9316:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775767824.049985:0:9316:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000400:00000200:3.0:1775767824.049988:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000020:00000001:2.0:1775767824.049991:0:9316:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:2.0:1775767824.049995:0:9316:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000400:00000200:3.0:1775767824.049995:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767824.049999:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000040:2.0:1775767824.050003:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767824.050005:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1775767824.050007:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:3.0:1775767824.050011:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dff480 x1862022294239360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767849 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775767824.050014:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767824.050022:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:3.0:1775767824.050027:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000020:00000040:2.0:1775767824.050028:0:9316:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00000040:3.0:1775767824.050033:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dff480 x1862022294239360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767849 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767824.050037:0:9316:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775767824.050045:0:9316:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767824.050047:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767824.050050:0:9316:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1775767824.050050:0:2399:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:2.0:1775767824.050057:0:9316:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767824.050058:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767824.050064:0:9316:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767824.050069:0:9316:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775767824.050071:0:9316:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775767824.050074:0:9316:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767824.050075:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767824.050076:0:9316:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767824.050080:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299750205 00000100:00000010:2.0:1775767824.050085:0:9316:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfed80. 00000100:00000010:3.0:1775767824.050087:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207981660. 00000020:00000040:2.0:1775767824.050090:0:9316:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:2.0:1775767824.050094:0:9316:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775767824.050096:0:9316:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767824.050096:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1775767824.050100:0:9316:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775767824.050103:0:9316:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1775767824.050104:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299750205 02000000:00000001:2.0:1775767824.050106:0:9316:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767824.050107:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207981600. 00000100:00000001:3.0:1775767824.050112:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 02000000:00000010:2.0:1775767824.050115:0:9316:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1f000. 00000100:00000001:3.0:1775767824.050115:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767824.050121:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767824.050122:0:9316:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767824.050123:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767824.050129:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767824/real 1775767824] req@ffff994207dff480 x1862022294239360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767824.050134:0:9316:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfed80 x1862022294239616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:3.0:1775767824.050138:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767824.050141:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767824.050145:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767824.050149:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767824.050152:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:2.0:1775767824.050153:0:9316:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767824.050177:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767824.050180:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dff480 x1862022294239360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767824.050186:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dff480 x1862022294239360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767824.050191:0:9316:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767824.050198:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767824.050201:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767824.050202:0:9316:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb9b9eae] to pc [ptlrpcd_rcv+-1] req@ffff994207dfed80 x1862022294239616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767824.050205:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767824.050208:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767824.050216:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dff480 x1862022294239360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767824.050225:0:9316:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767824.050228:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1775767824.050230:0:9316:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775767824.050235:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dff480 x1862022294239360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767849 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767824.050237:0:9316:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1775767824.050240:0:9316:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775767824.050244:0:9316:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775767824.050252:0:9316:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:3.0:1775767824.050256:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767824.050259:0:9316:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5076 00000100:00000001:3.0:1775767824.050263:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767824.050267:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000400:00000200:2.0:1775767824.050268:0:9316:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767824.050271:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1775767824.050276:0:9316:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:3.0:1775767824.050278:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1775767824.050280:0:9316:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1775767824.050282:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767824.050287:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000040:2.0:1775767824.050288:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767824.050292:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00080000:3.0:1775767824.050293:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767824.050297:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767824.050298:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767824.050303:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000040:3.0:1775767824.050304:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dff480 x1862022294239360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767849 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1775767824.050309:0:9316:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775767824.050313:0:9316:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:2.0:1775767824.050318:0:9316:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00100000:3.0:1775767824.050319:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294239360:192.168.202.156@tcp:250:kworker.0 00000020:00000040:2.0:1775767824.050322:0:9316:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:2.0:1775767824.050329:0:9316:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:3.0:1775767824.050331:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1775767824.050334:0:9316:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:3.0:1775767824.050335:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767824.050339:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dff480 x1862022294239360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767849 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775767824.050344:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767824.050351:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1775767824.050351:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767824.050356:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000040:2.0:1775767824.050357:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000010:3.0:1775767824.050359:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b0000. 00000100:00000001:2.0:1775767824.050362:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 02000000:00000001:3.0:1775767824.050364:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000040:2.0:1775767824.050367:0:9316:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000020:00000001:3.0:1775767824.050368:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767824.050370:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:3.0:1775767824.050374:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00080000:2.0:1775767824.050375:0:9316:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 02000000:00000010:3.0:1775767824.050379:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1ec00. 00000100:00000001:2.0:1775767824.050380:0:9316:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767824.050384:0:9316:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767824.050385:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767824.050387:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:2.0:1775767824.050388:0:9316:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000010:3.0:1775767824.050389:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dff480. 02000000:00000001:2.0:1775767824.050395:0:9316:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767824.050396:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767824.050398:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767824.050400:0:9316:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775767824.050401:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:2.0:1775767824.050405:0:9316:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:2.0:1775767824.050409:0:9316:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767824.050410:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767824.050411:0:9316:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767824.050413:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767824.050416:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767824.050417:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000010:2.0:1775767824.050419:0:9316:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfe300. 00000100:00000001:3.0:1775767824.050419:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767824.050424:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfce00 x1862022294239488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1775767824.050425:0:9316:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:2.0:1775767824.050431:0:9316:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1775767824.050433:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 02000000:00000001:2.0:1775767824.050435:0:9316:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767824.050436:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767824.050441:0:9316:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767824.050441:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767824.050444:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767824.050445:0:9316:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767824.050446:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767824.050449:0:9316:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767824.050450:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294239488:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767824.050456:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767824.050457:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000010:2.0:1775767824.050459:0:9316:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1ec00. 02000000:00000001:3.0:1775767824.050459:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767824.050464:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1775767824.050467:0:9316:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767824.050468:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b0000. 02000000:00000001:3.0:1775767824.050473:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767824.050477:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00100000:2.0:1775767824.050479:0:9316:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfe300 x1862022294239744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:3.0:1775767824.050479:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767824.050484:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767824.050489:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294239488, portal 10 00000100:00000001:3.0:1775767824.050493:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767824.050495:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611840000 : -112966097711616 : ffff994207dfce00) 00000100:00000001:2.0:1775767824.050502:0:9316:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775767824.050506:0:9316:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767824.050507:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfce00 x1862022294239488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767849 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767824.050516:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767824.050517:0:9316:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c661fef] to pc [ptlrpcd_rcv+-1] req@ffff994207dfe300 x1862022294239744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775767824.050522:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767824.050525:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767824.050529:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294239488, offset 0 00000400:00000200:3.0:1775767824.050536:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767824.050542:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767824.050544:0:9316:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767824.050548:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00080000:2.0:1775767824.050550:0:9316:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:3.0:1775767824.050553:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:2.0:1775767824.050558:0:9316:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000400:00000200:3.0:1775767824.050560:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000100:00000001:2.0:1775767824.050562:0:9316:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775767824.050566:0:9316:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:3.0:1775767824.050566:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000100:00000001:2.0:1775767824.050572:0:9316:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:3.0:1775767824.050573:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00080000:2.0:1775767824.050578:0:9316:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5076 00000400:00000200:3.0:1775767824.050583:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225044:225044:256:4294967295] 192.168.202.156@tcp LPNI seq info [225040:225040:8:4294967295] 00000400:00000200:3.0:1775767824.050589:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767824.050608:0:9316:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767824.050615:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000400:00000200:2.0:1775767824.050618:0:9316:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775767824.050625:0:9316:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000800:00000200:3.0:1775767824.050630:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00000040:2.0:1775767824.050634:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000800:00000010:3.0:1775767824.050637:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000100:00000001:2.0:1775767824.050639:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767824.050644:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000800:00000200:3.0:1775767824.050647:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:2.0:1775767824.050649:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000800:00000200:3.0:1775767824.050653:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000020:00000001:2.0:1775767824.050655:0:9316:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775767824.050658:0:9316:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:2.0:1775767824.050661:0:9316:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000800:00000200:3.1:1775767824.050664:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000040:2.0:1775767824.050665:0:9316:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000800:00000200:3.1:1775767824.050670:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000020:00000001:2.0:1775767824.050671:0:9316:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:2.0:1775767824.050678:0:9316:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000800:00000100:3.0:1775767824.050681:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767824.050685:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000100:00000040:2.0:1775767824.050687:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000200:3.0:1775767824.050691:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:2.0:1775767824.050692:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767824.050696:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767824.050701:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000200:3.0:1775767824.050702:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000020:00000040:2.0:1775767824.050706:0:9316:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000400:00000200:3.0:1775767824.050709:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000100:00080000:2.0:1775767824.050712:0:9316:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000400:00000010:3.0:1775767824.050714:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:2.0:1775767824.050715:0:9316:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767824.050718:0:2399:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:2.0:1775767824.050720:0:9316:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775767824.050724:0:9316:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000200:3.0:1775767824.050726:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfce00 x1862022294239488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767849 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1775767824.050729:0:9316:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767824.050734:0:9316:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775767824.050737:0:9316:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:2.0:1775767824.050743:0:9316:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767824.050747:0:9316:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767824.050748:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000010:2.0:1775767824.050752:0:9316:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfc700. 00000100:00000040:3.0:1775767824.050753:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfce00 x1862022294239488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767849 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1775767824.050757:0:9316:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:2.0:1775767824.050764:0:9316:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775767824.050767:0:9316:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767824.050770:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767824.050771:0:9316:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775767824.050775:0:9316:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775767824.050778:0:9316:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767824.050782:0:2399:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000010:2.0:1775767824.050784:0:9316:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1c800. 00000100:00000001:3.0:1775767824.050788:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767824.050789:0:9316:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767824.050795:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000100:00100000:2.0:1775767824.050800:0:9316:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfc700 x1862022294239872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:3.0:1775767824.050804:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767824.050808:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299748883 00000100:00000001:3.0:1775767824.050815:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299750205 00000100:00000001:2.0:1775767824.050820:0:9316:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767824.050821:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299748883 00000100:00000001:2.0:1775767824.050826:0:9316:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767824.050827:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299750205 00000100:00000001:3.0:1775767824.050832:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1775767824.050834:0:9316:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a176dd1] to pc [ptlrpcd_rcv+-1] req@ffff994207dfc700 x1862022294239872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767824.050836:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767824.050841:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767824.050850:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767824/real 1775767824] req@ffff994207dfce00 x1862022294239488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767824.050851:0:9316:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767824.050859:0:9316:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5086) 00000400:00000001:3.0:1775767824.050866:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767824.050869:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767824.050871:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767824.050874:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767824.050877:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767824.050881:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767824.050885:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfce00 x1862022294239488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767824.050895:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfce00 x1862022294239488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767824.050902:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767824.050904:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767824.050906:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767824.050908:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767824.050912:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfce00 x1862022294239488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767824.050919:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767824.050924:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfce00 x1862022294239488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767849 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767824.050932:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767824.050935:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767824.050937:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767824.050938:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767824.050940:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767824.050943:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767824.050946:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767824.050950:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767824.050952:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767824.050957:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfce00 x1862022294239488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767849 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767824.050967:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294239488:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767824.050972:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767824.050975:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfed80 x1862022294239616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767824.050990:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767824.050992:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767824.050994:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767824.050996:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767824.050999:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767824.051002:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294239616:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767824.051005:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767824.051008:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767824.051009:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767824.051012:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767824.051016:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b2800. 02000000:00000001:3.0:1775767824.051018:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767824.051022:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767824.051025:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767824.051028:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767824.051031:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294239616, portal 10 00000100:00000001:3.0:1775767824.051033:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767824.051035:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611848064 : -112966097703552 : ffff994207dfed80) 00000100:00000040:3.0:1775767824.051041:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfed80 x1862022294239616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767849 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767824.051049:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767824.051051:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767824.051053:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767824.051056:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294239616, offset 0 00000400:00000200:3.0:1775767824.051060:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767824.051064:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767824.051068:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767824.051071:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767824.051073:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767824.051076:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767824.051078:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767824.051082:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225045:225045:256:4294967295] 192.168.202.156@tcp LPNI seq info [225041:225041:8:4294967295] 00000400:00000200:3.0:1775767824.051086:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767824.051092:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767824.051097:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767824.051101:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767824.051106:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767824.051109:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775767824.051114:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767824.051116:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000100:3.0:1775767824.051121:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767824.051123:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767824.051126:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767824.051130:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767824.051133:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767824.051135:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767824.051138:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767824.051142:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfed80 x1862022294239616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767849 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767824.051151:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767824.051175:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfed80 x1862022294239616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767849 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767824.051184:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767824.051186:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767824.051190:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767824.051194:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:3.0:1775767824.051198:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767824.051201:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299748876 00000100:00000001:3.0:1775767824.051203:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299750205 00000100:00000001:3.0:1775767824.051206:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299748876 00000100:00000001:3.0:1775767824.051208:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299750205 00000100:00000001:3.0:1775767824.051210:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767824.051213:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767824.051215:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767824.051219:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767824/real 1775767824] req@ffff994207dfed80 x1862022294239616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767824.051228:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767824.051229:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767824.051231:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767824.051234:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767824.051236:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767824.051238:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767824.051242:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfed80 x1862022294239616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767824.051251:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfed80 x1862022294239616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767824.051258:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767824.051260:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767824.051262:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767824.051263:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767824.051267:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfed80 x1862022294239616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767824.051275:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767824.051279:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfed80 x1862022294239616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767849 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767824.051287:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767824.051290:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767824.051292:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767824.051294:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767824.051296:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767824.051299:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767824.051302:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767824.051306:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767824.051309:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767824.051314:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfed80 x1862022294239616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767849 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767824.051323:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294239616:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767824.051327:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767824.051330:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767824.051333:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfce00 x1862022294239488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767849 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767824.051341:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767824.051343:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767824.051345:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b0000. 02000000:00000001:3.0:1775767824.051348:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767824.051350:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767824.051351:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:3.0:1775767824.051354:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767824.051356:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1e800. 02000000:00000001:3.0:1775767824.051359:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767824.051361:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767824.051363:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfce00. 00000100:00000001:3.0:1775767824.051366:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767824.051368:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767824.051369:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767824.051372:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfed80 x1862022294239616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767849 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767824.051379:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767824.051381:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767824.051382:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b2800. 02000000:00000001:3.0:1775767824.051385:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767824.051386:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767824.051388:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:3.0:1775767824.051390:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767824.051392:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1f000. 02000000:00000001:3.0:1775767824.051395:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767824.051396:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767824.051398:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfed80. 00000100:00000001:3.0:1775767824.051400:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767824.051402:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767824.051404:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767824.051408:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767824.051410:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767824.051413:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767824.051414:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767824.051416:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767824.051420:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfe300 x1862022294239744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767824.051428:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767824.051430:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767824.051432:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767824.051435:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767824.051437:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767824.051441:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294239744:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767824.051444:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767824.051446:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767824.051448:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767824.051450:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767824.051453:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b2800. 02000000:00000001:3.0:1775767824.051456:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767824.051460:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767824.051463:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767824.051468:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767824.051471:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294239744, portal 4 00000100:00000001:3.0:1775767824.051473:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767824.051475:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611845376 : -112966097706240 : ffff994207dfe300) 00000100:00000040:3.0:1775767824.051481:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfe300 x1862022294239744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767849 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767824.051494:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767824.051497:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767824.051499:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767824.051503:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294239744, offset 0 00000400:00000200:3.0:1775767824.051508:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767824.051513:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767824.051517:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767824.051520:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767824.051523:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767824.051525:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767824.051528:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767824.051531:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225046:225046:256:4294967295] 192.168.202.156@tcp LPNI seq info [225042:225042:8:4294967295] 00000400:00000200:3.0:1775767824.051535:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767824.051542:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767824.051547:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767824.051550:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767824.051554:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767824.051557:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775767824.051562:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767824.051565:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000100:3.0:1775767824.051569:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767824.051571:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767824.051574:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767824.051578:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767824.051582:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767824.051584:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767824.051587:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767824.051590:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfe300 x1862022294239744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767849 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767824.051618:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767824.051622:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfe300 x1862022294239744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767849 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767824.051637:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767824.051639:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767824.051643:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767824.051647:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:3.0:1775767824.051651:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767824.051653:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299748899 00000100:00000001:3.0:1775767824.051656:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299750205 00000100:00000001:3.0:1775767824.051659:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299748899 00000100:00000001:3.0:1775767824.051661:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299750205 00000100:00000001:3.0:1775767824.051664:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767824.051666:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767824.051669:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767824.051673:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767824/real 1775767824] req@ffff994207dfe300 x1862022294239744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767824.051682:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767824.051684:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767824.051685:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767824.051688:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767824.051690:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767824.051692:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767824.051696:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfe300 x1862022294239744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767824.051706:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfe300 x1862022294239744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767824.051714:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767824.051715:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767824.051717:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767824.051719:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767824.051723:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfe300 x1862022294239744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767824.051732:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767824.051736:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfe300 x1862022294239744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767849 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767824.051745:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767824.051748:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767824.051750:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767824.051751:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767824.051753:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767824.051757:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767824.051760:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767824.051764:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767824.051766:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767824.051771:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfe300 x1862022294239744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767849 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767824.051781:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294239744:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767824.051785:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767824.051788:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfc700 x1862022294239872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767824.051795:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767824.051797:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767824.051800:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767824.051802:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767824.051804:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767824.051807:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294239872:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767824.051811:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767824.051812:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767824.051815:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767824.051817:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767824.051820:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b0000. 02000000:00000001:3.0:1775767824.051823:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767824.051827:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767824.051830:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767824.051833:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767824.051836:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294239872, portal 4 00000100:00000001:3.0:1775767824.051838:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767824.051840:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611838208 : -112966097713408 : ffff994207dfc700) 00000100:00000040:3.0:1775767824.051846:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfc700 x1862022294239872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767849 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767824.051854:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767824.051856:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767824.051859:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767824.051862:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294239872, offset 0 00000400:00000200:3.0:1775767824.051866:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767824.051870:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767824.051874:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767824.051877:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767824.051880:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767824.051883:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767824.051886:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767824.051889:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225047:225047:256:4294967295] 192.168.202.156@tcp LPNI seq info [225043:225043:8:4294967295] 00000400:00000200:3.0:1775767824.051894:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767824.051901:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767824.051906:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767824.051910:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767824.051915:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767824.051918:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775767824.051922:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767824.051925:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000100:3.0:1775767824.051930:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767824.051932:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767824.051935:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767824.051939:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767824.051942:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767824.051944:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767824.051946:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767824.051950:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfc700 x1862022294239872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767849 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767824.051959:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767824.051962:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfc700 x1862022294239872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767849 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767824.051970:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767824.051972:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767824.051975:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767824.051978:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:3.0:1775767824.051982:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767824.051984:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299748902 00000100:00000001:3.0:1775767824.051987:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299750205 00000100:00000001:3.0:1775767824.051990:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299748902 00000100:00000001:3.0:1775767824.051993:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299750205 00000100:00000001:3.0:1775767824.051995:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767824.051998:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767824.052000:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767824.052004:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767824/real 1775767824] req@ffff994207dfc700 x1862022294239872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767824.052014:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767824.052015:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767824.052017:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767824.052020:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767824.052022:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767824.052025:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767824.052028:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfc700 x1862022294239872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767824.052038:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfc700 x1862022294239872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767824.052046:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767824.052048:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767824.052050:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767824.052052:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767824.052055:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfc700 x1862022294239872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767824.052063:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767824.052067:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfc700 x1862022294239872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767849 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767824.052074:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767824.052076:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767824.052078:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767824.052080:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767824.052082:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767824.052086:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767824.052089:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767824.052092:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767824.052095:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767824.052099:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfc700 x1862022294239872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767849 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767824.052108:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294239872:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767824.052113:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767824.052115:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767824.052119:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfe300 x1862022294239744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767849 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767824.052126:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767824.052128:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767824.052130:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b2800. 02000000:00000001:3.0:1775767824.052133:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767824.052135:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767824.052136:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:3.0:1775767824.052139:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767824.052141:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1ec00. 02000000:00000001:3.0:1775767824.052144:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767824.052145:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767824.052146:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfe300. 00000100:00000001:3.0:1775767824.052149:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767824.052151:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767824.052172:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767824.052177:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfc700 x1862022294239872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767849 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767824.052185:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767824.052187:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767824.052189:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b0000. 02000000:00000001:3.0:1775767824.052192:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767824.052193:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767824.052195:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:3.0:1775767824.052197:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767824.052199:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1c800. 02000000:00000001:3.0:1775767824.052202:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767824.052203:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767824.052204:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfc700. 00000100:00000001:3.0:1775767824.052207:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767824.052209:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767824.052211:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767824.052214:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767824.052216:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767824.052218:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767824.052220:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767824.113374:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767824.113379:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767824.113388:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775767825.073224:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767825.073239:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767825.073243:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767825.073247:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767825.073253:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767825.073257:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767825.073258:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767825.073263:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767825.073266:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767825.073268:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767825.073270:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767825.073274:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767825.073275:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767825.073280:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767825.073284:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767825.073286:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767825.073288:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767825.073291:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767825.393239:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299750205, at 4299750208 00000100:00000010:1.0:1775767825.393254:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207981600. 00000100:00000001:3.0:1775767826.097238:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767826.097244:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767826.097247:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767826.097262:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:3.0:1775767826.097264:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767826.097270:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767826.097273:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767826.097274:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1775767826.097281:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767826.097282:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767826.097286:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767826.097292:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767826.097306:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767826.097309:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767826.097312:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767826.097313:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767826.097319:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767826.097324:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767826.161391:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767826.161394:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767826.161397:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775767827.121237:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767827.121244:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767827.121247:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767827.121255:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767827.121259:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767827.121264:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767827.121267:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767827.121270:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767827.121271:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767827.121274:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767827.121276:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767827.121276:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767827.121282:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767827.121286:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767827.121288:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767827.121290:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767827.121291:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767827.121293:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767828.145280:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767828.145283:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767828.145296:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767828.145297:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767828.145300:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767828.145304:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767828.145309:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:1.0:1775767828.145318:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767828.145318:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767828.145327:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767828.145327:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767828.145333:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767828.145333:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767828.145337:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767828.145341:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767828.145342:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767828.145346:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767828.145349:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767828.209361:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767828.209365:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767828.209368:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775767829.169306:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:3.0:1775767829.169306:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775767829.169315:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767829.169316:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:3.0:1775767829.169318:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775767829.169319:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767829.169323:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767829.169325:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775767829.169326:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767829.169327:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00080000:3.0:1775767829.169331:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775767829.169333:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767829.169334:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767829.169338:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775767829.169339:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767829.169341:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767829.169343:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767829.169344:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767829.169349:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775767829.169349:0:13471:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5081 00000100:00000001:2.0:1775767829.169354:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767829.169363:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000400:00000200:3.0:1775767829.169366:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767829.169367:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767829.169372:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767829.169378:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767829.169383:0:13471:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767829.169387:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767829.169392:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767829.169396:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767829.169400:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767829.169405:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767829.169408:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:3.0:1775767829.169412:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767829.169415:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:3.0:1775767829.169417:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:3.0:1775767829.169420:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:3.0:1775767829.169423:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767829.169426:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767829.169428:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767829.169430:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767829.169433:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:3.0:1775767829.169437:0:13471:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767829.169452:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767829.169456:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767829.169459:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767829.169463:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767829.169466:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775767829.169469:0:13471:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767829.169481:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8380. 00000020:00000040:3.0:1775767829.169486:0:13471:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:3.0:1775767829.169491:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767829.169493:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767829.169497:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767829.169501:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767829.169507:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767829.169519:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0000. 00000100:00000001:3.0:1775767829.169527:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767829.169559:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8380 x1862022294240000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767829.169571:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767829.169577:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767829.169583:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055f5621a] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8380 x1862022294240000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767829.169607:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767829.169611:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767829.169614:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775767829.169616:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767829.169618:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767829.169622:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767829.169625:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5081 00000100:00000001:2.0:1775767829.169631:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:3.0:1775767829.169631:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767829.169636:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000400:00000200:3.0:1775767829.169637:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:2.0:1775767829.169639:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00080000:3.0:1775767829.169643:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775767829.169648:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a8380 x1862022294240000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775767829.169650:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767829.169656:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767829.169661:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767829.169663:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767829.169666:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:2.0:1775767829.169667:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767829.169672:0:13471:0:(genops.c:811:class_conn2export()) Process entered 02000000:00000001:2.0:1775767829.169673:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:3.0:1775767829.169674:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 02000000:00000001:2.0:1775767829.169676:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1775767829.169677:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000001:2.0:1775767829.169681:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775767829.169682:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000100:00100000:2.0:1775767829.169686:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294240000:192.168.202.156@tcp:250:kworker.0 00000020:00000001:3.0:1775767829.169687:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000001:2.0:1775767829.169692:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000001:3.0:1775767829.169693:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 02000000:00000001:2.0:1775767829.169696:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767829.169699:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767829.169700:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 02000000:00000001:2.0:1775767829.169703:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1775767829.169706:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767829.169709:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767829.169712:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767829.169714:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 02000000:00000010:2.0:1775767829.169717:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1fc00. 00000100:00080000:3.0:1775767829.169718:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 02000000:00000001:2.0:1775767829.169721:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767829.169722:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767829.169727:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:2.0:1775767829.169730:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767829.169734:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39c60. 02000000:00000001:3.0:1775767829.169734:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767829.169740:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767829.169742:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000010:2.0:1775767829.169743:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f220. 00000008:00100000:3.0:1775767829.169747:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00000200:2.0:1775767829.169751:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294240000, portal 25 00000008:00000001:3.0:1775767829.169752:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767829.169757:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767829.169757:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767829.169759:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640853376 : -112966068698240 : ffff9942099a8380) 00000100:00000010:3.0:1775767829.169764:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8700. 00000100:00000040:2.0:1775767829.169766:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a8380 x1862022294240000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767854 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767829.169774:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000040:3.0:1775767829.169775:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000100:00000040:2.0:1775767829.169779:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767829.169782:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 02000000:00000001:3.0:1775767829.169784:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000200:2.0:1775767829.169787:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294240000, offset 0 02000000:00000001:3.0:1775767829.169787:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767829.169791:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767829.169794:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:2.0:1775767829.169798:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 02000000:00000001:3.0:1775767829.169798:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767829.169805:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2800. 00000400:00000200:2.0:1775767829.169808:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767829.169811:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767829.169815:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00100000:3.0:1775767829.169820:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8700 x1862022294240128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1775767829.169821:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767829.169828:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767829.169835:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1775767829.169837:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:2.0:1775767829.169839:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00000001:3.0:1775767829.169842:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767829.169846:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225048:225048:256:4294967295] 192.168.202.156@tcp LPNI seq info [225044:225044:8:4294967295] 00000100:00000040:3.0:1775767829.169850:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000faa25e3b] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8700 x1862022294240128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1775767829.169856:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767829.169873:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767829.169878:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:2.0:1775767829.169879:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000100:00080000:3.0:1775767829.169885:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775767829.169888:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000800:00000200:2.0:1775767829.169891:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00080000:3.0:1775767829.169891:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767829.169896:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000800:00000010:2.0:1775767829.169899:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3300. 00000100:00080000:3.0:1775767829.169901:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5081 00000400:00000200:3.0:1775767829.169907:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000800:00000200:2.0:1775767829.169911:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000400:00000200:3.0:1775767829.169915:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000800:00000200:2.0:1775767829.169916:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000100:00080000:3.0:1775767829.169920:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000800:00000200:2.1:1775767829.169924:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000040:3.0:1775767829.169927:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000800:00000200:2.1:1775767829.169928:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000100:00000001:3.0:1775767829.169931:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000100:2.0:1775767829.169935:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767829.169937:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3300. 00000400:00000200:2.0:1775767829.169943:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775767829.169948:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775767829.169956:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000100:00000040:3.0:1775767829.169956:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767829.169961:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000010:2.0:1775767829.169962:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000020:00000001:3.0:1775767829.169963:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767829.169966:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:3.0:1775767829.169969:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:2.0:1775767829.169970:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000020:00000040:3.0:1775767829.169971:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:3.0:1775767829.169975:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000200:2.0:1775767829.169980:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a8380 x1862022294240000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767854 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775767829.169980:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:3.0:1775767829.169987:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767829.169994:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767829.169998:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767829.170002:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1775767829.170002:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767829.170006:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00000040:2.0:1775767829.170009:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a8380 x1862022294240000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767854 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767829.170012:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767829.170017:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767829.170022:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767829.170027:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767829.170028:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767829.170033:0:2399:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:3.0:1775767829.170033:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767829.170036:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767829.170039:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1775767829.170041:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1775767829.170041:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767829.170046:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767829.170049:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000010:3.0:1775767829.170050:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a9180. 00000100:00000001:2.0:1775767829.170053:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299755325 00000020:00000040:3.0:1775767829.170054:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:3.0:1775767829.170060:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767829.170061:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767829.170063:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000010:2.0:1775767829.170064:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d60c0. 02000000:00000001:3.0:1775767829.170065:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767829.170069:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767829.170074:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0c00. 00000100:00000001:2.0:1775767829.170075:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1775767829.170080:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767829.170084:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299755325 00000100:00100000:3.0:1775767829.170088:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a9180 x1862022294240256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000010:2.0:1775767829.170091:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d6780. 00000100:00000001:2.0:1775767829.170098:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1775767829.170101:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775767829.170103:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767829.170106:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767829.170107:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767829.170111:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000040:3.0:1775767829.170113:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bdc2586] to pc [ptlrpcd_rcv+-1] req@ffff9942099a9180 x1862022294240256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767829.170118:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767829/real 1775767829] req@ffff9942099a8380 x1862022294240000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767829.170134:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767829.170138:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000001:2.0:1775767829.170139:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767829.170142:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:3.0:1775767829.170145:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000400:00000010:2.0:1775767829.170147:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39c60. 00000100:00000001:3.0:1775767829.170148:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:2.0:1775767829.170150:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f220 00000100:00080000:3.0:1775767829.170151:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000010:2.0:1775767829.170153:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f220. 00000100:00000001:2.0:1775767829.170173:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767829.170176:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a8380 x1862022294240000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767829.170179:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767829.170186:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5081 00000100:00000200:2.0:1775767829.170190:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a8380 x1862022294240000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1775767829.170194:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767829.170202:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:2.0:1775767829.170203:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767829.170207:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775767829.170207:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1775767829.170211:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767829.170213:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767829.170214:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00100000:2.0:1775767829.170219:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a8380 x1862022294240000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767829.170219:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767829.170225:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767829.170230:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:2.0:1775767829.170232:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1775767829.170235:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767829.170238:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000100:00000040:2.0:1775767829.170240:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a8380 x1862022294240000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767854 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775767829.170241:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767829.170244:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:3.0:1775767829.170249:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:2.0:1775767829.170251:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:3.0:1775767829.170252:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:2.0:1775767829.170255:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767829.170257:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:3.0:1775767829.170259:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767829.170260:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767829.170265:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767829.170265:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767829.170268:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767829.170269:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767829.170273:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1775767829.170273:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767829.170278:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:2.0:1775767829.170281:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00080000:3.0:1775767829.170284:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775767829.170285:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1775767829.170288:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767829.170292:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a8380 x1862022294240000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767854 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1775767829.170292:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767829.170297:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767829.170302:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767829.170304:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294240000:192.168.202.156@tcp:250:kworker.0 00000100:00000001:3.0:1775767829.170306:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767829.170310:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000100:00000001:2.0:1775767829.170311:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1775767829.170313:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767829.170314:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1775767829.170316:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767829.170319:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a8380 x1862022294240000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767854 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1775767829.170322:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ab480. 00000100:00000001:2.0:1775767829.170327:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 00000020:00000040:3.0:1775767829.170327:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:2.0:1775767829.170330:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1775767829.170332:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000010:2.0:1775767829.170334:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1fc00. 02000000:00000001:3.0:1775767829.170335:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767829.170338:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1775767829.170338:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000020:00000001:2.0:1775767829.170342:0:2399:0:(genops.c:1236:class_import_put()) Process entered 02000000:00000001:3.0:1775767829.170342:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767829.170345:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775767829.170346:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:2.0:1775767829.170351:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767829.170353:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2c00. 02000000:00000010:2.0:1775767829.170354:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b0000. 02000000:00000001:2.0:1775767829.170360:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:3.0:1775767829.170360:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767829.170363:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767829.170365:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a8380. 00000100:00000001:2.0:1775767829.170369:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00100000:3.0:1775767829.170369:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ab480 x1862022294240384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775767829.170372:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767829.170376:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767829.170385:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775767829.170387:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767829.170387:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767829.170390:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767829.170394:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1775767829.170394:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000067a0ba74] to pc [ptlrpcd_rcv+-1] req@ffff9942099ab480 x1862022294240384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767829.170397:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767829.170399:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767829.170405:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a8700 x1862022294240128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767829.170412:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767829.170415:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767829.170417:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767829.170417:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:2.0:1775767829.170421:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767829.170424:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1775767829.170424:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 02000000:00000001:2.0:1775767829.170428:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767829.170428:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767829.170431:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775767829.170435:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294240128:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767829.170436:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767829.170441:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5081 00000100:00000001:2.0:1775767829.170443:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767829.170447:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000400:00000200:3.0:1775767829.170450:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:2.0:1775767829.170451:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767829.170456:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:3.0:1775767829.170456:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767829.170460:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 02000000:00000010:2.0:1775767829.170461:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1fc00. 00000100:00000040:3.0:1775767829.170464:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 02000000:00000001:2.0:1775767829.170466:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767829.170468:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767829.170472:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767829.170476:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000200:2.0:1775767829.170477:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:3.0:1775767829.170481:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000400:00000010:2.0:1775767829.170483:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39c60. 00000020:00000040:3.0:1775767829.170484:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:3.0:1775767829.170486:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767829.170489:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000400:00000010:2.0:1775767829.170490:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f220. 00000020:00000001:3.0:1775767829.170491:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:3.0:1775767829.170495:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000200:2.0:1775767829.170496:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294240128, portal 10 00000100:00000040:3.0:1775767829.170500:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767829.170501:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767829.170504:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775767829.170505:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640854272 : -112966068697344 : ffff9942099a8700) 00000100:00000040:3.0:1775767829.170509:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767829.170512:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767829.170514:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00000040:2.0:1775767829.170518:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a8700 x1862022294240128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767854 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767829.170520:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767829.170525:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767829.170529:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767829.170532:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767829.170537:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1775767829.170537:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767829.170541:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000040:2.0:1775767829.170557:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000008:00100000:3.0:1775767829.170560:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000400:00000010:2.0:1775767829.170561:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000008:00000001:3.0:1775767829.170567:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1775767829.170568:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294240128, offset 0 00000100:00000001:3.0:1775767829.170571:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767829.170575:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767829.170579:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000010:3.0:1775767829.170579:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8e00. 00000020:00000040:3.0:1775767829.170584:0:13471:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00000400:00000200:2.0:1775767829.170586:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 02000000:00000001:3.0:1775767829.170591:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000200:2.0:1775767829.170592:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:3.0:1775767829.170594:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767829.170597:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000100:00000001:3.0:1775767829.170599:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:2.0:1775767829.170601:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 02000000:00000001:3.0:1775767829.170602:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:2.0:1775767829.170605:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 02000000:00000001:3.0:1775767829.170607:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767829.170609:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225049:225049:256:4294967295] 192.168.202.156@tcp LPNI seq info [225045:225045:8:4294967295] 00000400:00000200:2.0:1775767829.170613:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000010:3.0:1775767829.170614:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b3400. 00000400:00000200:2.0:1775767829.170620:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000100:00000001:3.0:1775767829.170621:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1775767829.170627:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00100000:3.0:1775767829.170632:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8e00 x1862022294240512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000010:2.0:1775767829.170633:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3300. 00000800:00000200:2.0:1775767829.170643:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775767829.170648:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000100:00000001:3.0:1775767829.170650:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000200:2.1:1775767829.170654:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:3.0:1775767829.170655:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775767829.170659:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000100:00000040:3.0:1775767829.170664:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000864a6866] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8e00 x1862022294240512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1775767829.170667:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767829.170671:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3300. 00000400:00000200:2.0:1775767829.170676:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775767829.170683:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000100:00000001:3.0:1775767829.170683:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767829.170686:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775767829.170688:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000040:3.0:1775767829.170690:0:13471:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5091) 00000100:00000001:2.0:1775767829.170691:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767829.170697:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a8700 x1862022294240128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767854 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767829.170711:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767829.170714:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a8700 x1862022294240128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767854 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767829.170721:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767829.170723:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775767829.170726:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767829.170732:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:2.0:1775767829.170736:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767829.170738:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299754003 00000100:00000001:2.0:1775767829.170740:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299755325 00000100:00000001:2.0:1775767829.170742:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299754003 00000100:00000001:2.0:1775767829.170745:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299755325 00000100:00000001:2.0:1775767829.170747:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767829.170749:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767829.170751:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767829.170754:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767829/real 1775767829] req@ffff9942099a8700 x1862022294240128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775767829.170762:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767829.170763:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767829.170765:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39c60. 00000400:00000200:2.0:1775767829.170767:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f220 00000400:00000010:2.0:1775767829.170769:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f220. 00000100:00000001:2.0:1775767829.170770:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767829.170774:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a8700 x1862022294240128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767829.170782:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a8700 x1862022294240128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767829.170789:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767829.170791:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767829.170793:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767829.170795:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767829.170798:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a8700 x1862022294240128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767829.170804:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767829.170808:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a8700 x1862022294240128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767854 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767829.170814:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767829.170816:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767829.170818:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767829.170819:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767829.170821:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767829.170824:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767829.170828:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767829.170830:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767829.170833:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767829.170836:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a8700 x1862022294240128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767854 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767829.170844:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294240128:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775767829.170848:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767829.170850:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a9180 x1862022294240256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767829.170856:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767829.170858:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767829.170860:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767829.170862:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767829.170864:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767829.170867:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294240256:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775767829.170870:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767829.170871:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767829.170873:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767829.170875:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775767829.170879:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1e400. 02000000:00000001:2.0:1775767829.170880:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767829.170883:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767829.170886:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39c60. 00000400:00000010:2.0:1775767829.170889:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f220. 00000100:00000200:2.0:1775767829.170892:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294240256, portal 10 00000100:00000001:2.0:1775767829.170894:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767829.170895:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640856960 : -112966068694656 : ffff9942099a9180) 00000100:00000040:2.0:1775767829.170901:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a9180 x1862022294240256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767854 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767829.170907:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767829.170909:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767829.170912:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775767829.170914:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294240256, offset 0 00000400:00000200:2.0:1775767829.170918:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767829.170922:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767829.170926:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767829.170929:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767829.170931:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767829.170934:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767829.170936:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767829.170940:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225050:225050:256:4294967295] 192.168.202.156@tcp LPNI seq info [225046:225046:8:4294967295] 00000400:00000200:2.0:1775767829.170945:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767829.170951:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767829.170956:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767829.170959:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3300. 00000800:00000200:2.0:1775767829.170963:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775767829.170965:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775767829.170968:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767829.170969:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1775767829.170972:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767829.170973:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3300. 00000400:00000200:2.0:1775767829.170976:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775767829.170978:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775767829.170980:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775767829.170981:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775767829.170983:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767829.170986:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a9180 x1862022294240256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767854 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767829.170991:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767829.170993:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a9180 x1862022294240256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767854 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767829.170998:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767829.170999:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775767829.171001:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767829.171003:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:2.0:1775767829.171006:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767829.171007:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299753995 00000100:00000001:2.0:1775767829.171009:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299755325 00000100:00000001:2.0:1775767829.171010:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299753995 00000100:00000001:2.0:1775767829.171012:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299755325 00000100:00000001:2.0:1775767829.171013:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767829.171015:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767829.171016:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767829.171018:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767829/real 1775767829] req@ffff9942099a9180 x1862022294240256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775767829.171024:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767829.171025:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767829.171026:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39c60. 00000400:00000200:2.0:1775767829.171028:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f220 00000400:00000010:2.0:1775767829.171029:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f220. 00000100:00000001:2.0:1775767829.171030:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767829.171032:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a9180 x1862022294240256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767829.171037:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a9180 x1862022294240256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767829.171041:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767829.171042:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767829.171043:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767829.171044:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767829.171046:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a9180 x1862022294240256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767829.171049:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767829.171052:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a9180 x1862022294240256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767854 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767829.171056:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767829.171058:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767829.171059:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767829.171060:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767829.171061:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767829.171063:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767829.171065:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767829.171066:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767829.171068:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767829.171070:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a9180 x1862022294240256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767854 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767829.171080:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294240256:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775767829.171083:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767829.171084:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767829.171086:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a8700 x1862022294240128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767854 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767829.171089:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767829.171090:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767829.171092:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1fc00. 02000000:00000001:2.0:1775767829.171093:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767829.171094:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767829.171095:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:2.0:1775767829.171096:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767829.171097:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b2800. 02000000:00000001:2.0:1775767829.171099:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767829.171099:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767829.171101:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a8700. 00000100:00000001:2.0:1775767829.171102:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767829.171103:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767829.171104:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767829.171106:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a9180 x1862022294240256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767854 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767829.171110:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767829.171111:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767829.171111:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1e400. 02000000:00000001:2.0:1775767829.171114:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767829.171114:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767829.171115:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:2.0:1775767829.171116:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767829.171117:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b0c00. 02000000:00000001:2.0:1775767829.171118:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767829.171119:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767829.171120:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a9180. 00000100:00000001:2.0:1775767829.171122:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767829.171123:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767829.171125:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767829.171127:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767829.171128:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767829.171130:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775767829.171131:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767829.171132:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767829.171134:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ab480 x1862022294240384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767829.171138:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767829.171139:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767829.171141:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767829.171142:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767829.171144:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767829.171146:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ab480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294240384:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767829.171148:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767829.171149:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767829.171150:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767829.171151:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775767829.171153:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1e400. 02000000:00000001:2.0:1775767829.171170:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767829.171174:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767829.171176:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39c60. 00000400:00000010:2.0:1775767829.171179:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f220. 00000100:00000200:2.0:1775767829.171181:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294240384, portal 4 00000100:00000001:2.0:1775767829.171182:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767829.171183:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640865920 : -112966068685696 : ffff9942099ab480) 00000100:00000040:2.0:1775767829.171188:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ab480 x1862022294240384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767854 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767829.171193:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767829.171194:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767829.171195:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775767829.171197:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294240384, offset 0 00000400:00000200:2.0:1775767829.171200:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767829.171203:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767829.171205:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767829.171207:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767829.171209:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767829.171211:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767829.171213:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767829.171215:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225051:225051:256:4294967295] 192.168.202.156@tcp LPNI seq info [225047:225047:8:4294967295] 00000400:00000200:2.0:1775767829.171218:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767829.171222:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767829.171225:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767829.171227:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3300. 00000800:00000200:2.0:1775767829.171230:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775767829.171232:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775767829.171235:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767829.171236:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1775767829.171239:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767829.171241:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3300. 00000400:00000200:2.0:1775767829.171242:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775767829.171245:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775767829.171247:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775767829.171248:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775767829.171250:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767829.171252:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ab480 x1862022294240384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767854 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767829.171257:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767829.171259:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ab480 x1862022294240384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767854 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767829.171263:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767829.171264:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775767829.171266:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767829.171268:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:2.0:1775767829.171270:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767829.171272:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299754019 00000100:00000001:2.0:1775767829.171273:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299755325 00000100:00000001:2.0:1775767829.171275:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299754019 00000100:00000001:2.0:1775767829.171277:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299755325 00000100:00000001:2.0:1775767829.171278:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767829.171279:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767829.171280:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767829.171282:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767829/real 1775767829] req@ffff9942099ab480 x1862022294240384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775767829.171286:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767829.171287:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767829.171289:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39c60. 00000400:00000200:2.0:1775767829.171290:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f220 00000400:00000010:2.0:1775767829.171291:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f220. 00000100:00000001:2.0:1775767829.171293:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767829.171295:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ab480 x1862022294240384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767829.171300:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ab480 x1862022294240384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767829.171303:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767829.171304:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767829.171305:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767829.171306:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767829.171308:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ab480 x1862022294240384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767829.171312:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767829.171314:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ab480 x1862022294240384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767854 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767829.171318:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767829.171320:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767829.171322:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767829.171322:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767829.171324:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767829.171327:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767829.171329:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767829.171331:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767829.171332:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767829.171334:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ab480 x1862022294240384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767854 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767829.171339:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ab480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294240384:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767829.171341:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767829.171344:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a8e00 x1862022294240512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767829.171348:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767829.171349:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767829.171350:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767829.171351:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767829.171352:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767829.171354:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294240512:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767829.171356:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767829.171357:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767829.171358:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767829.171359:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775767829.171361:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1fc00. 02000000:00000001:2.0:1775767829.171372:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767829.171374:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767829.171379:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39c60. 00000400:00000010:2.0:1775767829.171381:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f220. 00000100:00000200:2.0:1775767829.171383:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294240512, portal 4 00000100:00000001:2.0:1775767829.171385:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767829.171386:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640856064 : -112966068695552 : ffff9942099a8e00) 00000100:00000040:2.0:1775767829.171390:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a8e00 x1862022294240512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767854 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767829.171394:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767829.171396:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767829.171397:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775767829.171399:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294240512, offset 0 00000400:00000200:2.0:1775767829.171402:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767829.171404:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767829.171406:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767829.171407:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767829.171409:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767829.171411:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767829.171412:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767829.171414:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225052:225052:256:4294967295] 192.168.202.156@tcp LPNI seq info [225048:225048:8:4294967295] 00000400:00000200:2.0:1775767829.171417:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767829.171421:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767829.171424:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767829.171426:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3300. 00000800:00000200:2.0:1775767829.171428:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775767829.171430:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775767829.171434:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767829.171436:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1775767829.171441:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767829.171443:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3300. 00000400:00000200:2.0:1775767829.171446:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775767829.171449:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775767829.171452:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775767829.171453:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775767829.171456:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767829.171459:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a8e00 x1862022294240512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767854 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767829.171474:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767829.171477:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a8e00 x1862022294240512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767854 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767829.171485:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767829.171487:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775767829.171490:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767829.171493:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:2.0:1775767829.171496:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767829.171498:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299754022 00000100:00000001:2.0:1775767829.171500:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299755325 00000100:00000001:2.0:1775767829.171501:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299754022 00000100:00000001:2.0:1775767829.171504:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299755325 00000100:00000001:2.0:1775767829.171506:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767829.171509:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767829.171511:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767829.171514:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767829/real 1775767829] req@ffff9942099a8e00 x1862022294240512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775767829.171523:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767829.171525:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767829.171526:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39c60. 00000400:00000200:2.0:1775767829.171529:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f220 00000400:00000010:2.0:1775767829.171531:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f220. 00000100:00000001:2.0:1775767829.171533:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767829.171537:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a8e00 x1862022294240512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767829.171564:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a8e00 x1862022294240512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767829.171571:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767829.171573:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767829.171574:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767829.171575:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767829.171578:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a8e00 x1862022294240512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767829.171583:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767829.171585:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a8e00 x1862022294240512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767854 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767829.171589:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767829.171591:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767829.171592:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767829.171592:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767829.171593:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767829.171597:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767829.171599:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767829.171601:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767829.171602:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767829.171604:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a8e00 x1862022294240512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767854 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767829.171609:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294240512:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767829.171612:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767829.171613:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767829.171615:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ab480 x1862022294240384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767854 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767829.171619:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767829.171620:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767829.171621:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1e400. 02000000:00000001:2.0:1775767829.171623:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767829.171624:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767829.171625:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:2.0:1775767829.171627:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767829.171628:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b2c00. 02000000:00000001:2.0:1775767829.171630:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767829.171631:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767829.171632:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ab480. 00000100:00000001:2.0:1775767829.171634:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767829.171635:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767829.171636:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767829.171638:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a8e00 x1862022294240512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767854 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767829.171642:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767829.171643:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767829.171644:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1fc00. 02000000:00000001:2.0:1775767829.171645:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767829.171646:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767829.171646:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:2.0:1775767829.171647:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767829.171648:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b3400. 02000000:00000001:2.0:1775767829.171649:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767829.171650:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767829.171651:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a8e00. 00000100:00000001:2.0:1775767829.171652:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767829.171653:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767829.171654:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767829.171656:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767829.171657:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767829.171658:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775767829.171659:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767830.193283:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767830.193283:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767830.193290:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767830.193291:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767830.193292:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767830.193296:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767830.193303:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:2.0:1775767830.193304:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767830.193313:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767830.193313:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767830.193321:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767830.193322:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767830.193338:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767830.193345:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767830.193351:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767830.193351:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767830.193354:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767830.193359:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767830.257364:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767830.257369:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767830.257374:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775767830.513255:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299755325, at 4299755328 00000100:00000010:1.0:1775767830.513267:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d6780. 00000100:00000001:2.0:1775767831.217268:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767831.217275:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767831.217278:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767831.217285:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775767831.217290:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767831.217294:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767831.217297:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767831.217300:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767831.217304:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767831.217314:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767831.217323:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767831.217325:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767831.217328:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767831.217330:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767831.217335:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767831.217336:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775767831.217342:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767831.217343:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767831.217352:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767831.217352:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767831.217357:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767831.217358:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767831.217362:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767831.217367:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767831.217370:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767831.217374:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767831.217377:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767832.241259:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767832.241259:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767832.241266:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767832.241267:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767832.241268:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767832.241270:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767832.241274:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:2.0:1775767832.241275:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767832.241279:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767832.241282:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767832.241284:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767832.241284:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767832.241286:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767832.241292:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767832.241293:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767832.241296:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767832.241296:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767832.241312:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767832.241312:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767832.241313:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767832.241317:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767832.241324:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767832.241325:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767832.241334:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767832.241337:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767832.241340:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767832.241344:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767832.305347:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767832.305351:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767832.305355:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775767833.265260:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767833.265260:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767833.265264:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767833.265267:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767833.265267:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767833.265274:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767833.265275:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767833.265275:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767833.265277:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767833.265289:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:2.0:1775767833.265290:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775767833.265290:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767833.265304:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767833.265305:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767833.265305:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767833.265312:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767833.265314:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767833.265314:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767833.265318:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767833.265322:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767833.265325:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767833.265328:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767833.265331:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767833.265335:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767833.265336:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767833.265338:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767833.265342:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767834.289319:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:3.0:1775767834.289327:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775767834.289328:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767834.289333:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767834.289349:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00080000:3.0:1775767834.289349:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775767834.289357:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775767834.289358:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1775767834.289365:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775767834.289368:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767834.289376:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767834.289377:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775767834.289386:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767834.289396:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767834.289396:0:13471:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5086 00000400:00000200:3.0:1775767834.289419:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767834.289427:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767834.289430:0:13471:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1775767834.289434:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000040:3.0:1775767834.289435:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767834.289441:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767834.289442:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775767834.289444:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767834.289449:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000040:3.0:1775767834.289449:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767834.289460:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767834.289460:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:1.0:1775767834.289466:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1775767834.289468:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:1.0:1775767834.289470:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000040:3.0:1775767834.289472:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000100:00000001:1.0:1775767834.289477:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1775767834.289479:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:1.0:1775767834.289481:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775767834.289482:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:3.0:1775767834.289488:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:3.0:1775767834.289491:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:3.0:1775767834.289494:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767834.289514:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767834.289519:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767834.289523:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767834.289526:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:3.0:1775767834.289545:0:13471:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767834.289547:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767834.289549:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767834.289552:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767834.289554:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767834.289555:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775767834.289558:0:13471:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767834.289563:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aad80. 00000020:00000040:3.0:1775767834.289566:0:13471:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:3.0:1775767834.289570:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767834.289572:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767834.289574:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767834.289577:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767834.289580:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767834.289587:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2000. 00000100:00000001:3.0:1775767834.289593:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767834.289602:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aad80 x1862022294240640/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767834.289610:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767834.289613:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767834.289617:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000045ccf08] to pc [ptlrpcd_rcv+-1] req@ffff9942099aad80 x1862022294240640/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767834.289636:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767834.289639:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767834.289642:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775767834.289644:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767834.289645:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767834.289648:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767834.289650:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5086 00000400:00000200:3.0:1775767834.289655:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767834.289659:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767834.289661:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767834.289665:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767834.289670:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767834.289674:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767834.289679:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767834.289683:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767834.289686:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:3.0:1775767834.289688:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767834.289748:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:3.0:1775767834.289751:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:3.0:1775767834.289757:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:3.0:1775767834.289762:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767834.289766:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767834.289770:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767834.289774:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767834.289778:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00080000:3.0:1775767834.289781:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767834.289784:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767834.289786:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767834.289791:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767834.289797:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767834.289800:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000800:00000200:0.1:1775767834.289869:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000008:00100000:3.0:1775767834.289929:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775767834.289936:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767834.289939:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767834.289944:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aa680. 00000020:00000040:3.0:1775767834.289950:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:3.0:1775767834.289954:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767834.289955:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767834.289957:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767834.289959:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767834.289961:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767834.289966:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0400. 00000100:00000001:3.0:1775767834.289969:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767834.289975:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aa680 x1862022294240768/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767834.290046:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767834.290049:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767834.290054:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006cb1597d] to pc [ptlrpcd_rcv+-1] req@ffff9942099aa680 x1862022294240768/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767834.290066:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767834.290068:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767834.290071:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775767834.290072:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767834.290073:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767834.290076:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767834.290078:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5086 00000400:00000200:3.0:1775767834.290082:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767834.290086:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767834.290088:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767834.290090:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767834.290094:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767834.290098:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767834.290100:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767834.290103:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767834.290106:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:3.0:1775767834.290108:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767834.290111:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:3.0:1775767834.290114:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:3.0:1775767834.290120:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:3.0:1775767834.290126:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767834.290133:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767834.290139:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767834.290144:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767834.290149:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:3.0:1775767834.290154:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767834.290190:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767834.290194:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767834.290200:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767834.290205:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767834.290209:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767834.290213:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775767834.290217:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767834.290221:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767834.290226:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ab800. 00000020:00000040:3.0:1775767834.290231:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:3.0:1775767834.290237:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767834.290240:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767834.290244:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767834.290248:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767834.290252:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767834.290259:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b1000. 00000100:00000001:3.0:1775767834.290266:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767834.290275:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ab800 x1862022294240896/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767834.290299:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767834.290302:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767834.290310:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000089df80c0] to pc [ptlrpcd_rcv+-1] req@ffff9942099ab800 x1862022294240896/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767834.290329:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767834.290334:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767834.290341:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775767834.290344:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767834.290348:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767834.290353:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767834.290357:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5086 00000400:00000200:3.0:1775767834.290367:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767834.290378:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767834.290385:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767834.290394:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767834.290399:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767834.290404:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767834.290409:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767834.290415:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767834.290417:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:3.0:1775767834.290421:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767834.290425:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:3.0:1775767834.290429:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:3.0:1775767834.290434:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:3.0:1775767834.290443:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000800:00000200:0.1:1775767834.290446:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:3.0:1775767834.290448:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767834.290453:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767834.290458:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767834.290464:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:3.0:1775767834.290470:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767834.290475:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767834.290480:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767834.290486:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767834.290491:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767834.290514:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767834.290518:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:3.0:1775767834.290524:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767834.290526:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767834.290532:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aa300. 00000020:00000040:3.0:1775767834.290537:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:3.0:1775767834.290543:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767834.290546:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767834.290550:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767834.290553:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767834.290557:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767834.290564:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b3400. 00000100:00000001:3.0:1775767834.290572:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767834.290582:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aa300 x1862022294241024/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767834.290600:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767834.290604:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767834.290610:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f33bdf15] to pc [ptlrpcd_rcv+-1] req@ffff9942099aa300 x1862022294241024/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767834.290630:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767834.290634:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767834.290641:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775767834.290645:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767834.290649:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767834.290655:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767834.290658:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5086 00000400:00000200:3.0:1775767834.290668:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767834.290675:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767834.290680:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000800:00000200:0.1:1775767834.290741:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767834.290894:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000040:3.0:1775767834.291030:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767834.291033:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767834.291035:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767834.291037:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767834.291039:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767834.291041:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:3.0:1775767834.291042:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767834.291044:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:3.0:1775767834.291047:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:3.0:1775767834.291049:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:3.0:1775767834.291052:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767834.291054:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767834.291056:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767834.291059:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767834.291062:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:3.0:1775767834.291065:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767834.291068:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767834.291071:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767834.291074:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767834.291077:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767834.291079:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767834.291082:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:3.0:1775767834.291085:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767834.291086:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767834.291091:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8a80. 00000020:00000040:3.0:1775767834.291094:0:13471:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:3.0:1775767834.291096:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767834.291098:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767834.291099:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767834.291101:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767834.291103:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767834.291108:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2c00. 00000100:00000001:3.0:1775767834.291112:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767834.291118:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8a80 x1862022294241152/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767834.291126:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767834.291129:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767834.291133:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006c3265af] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8a80 x1862022294241152/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767834.291150:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767834.291155:0:13471:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5096) 00000800:00000200:0.1:1775767834.291391:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000001:2.0:1775767834.353360:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767834.353364:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767834.353366:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775767835.313324:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767835.313328:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767835.313329:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767835.313332:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767835.313335:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767835.313338:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767835.313340:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767835.313341:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767835.313343:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767835.313348:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767835.313352:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767835.313353:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767835.313357:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775767835.313361:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767835.313364:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767835.313366:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767835.313368:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767835.313371:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767835.633253:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299760445, at 4299760448 00000100:00000010:1.0:1775767835.633270:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a39c0. 00000100:00000001:2.0:1775767836.337249:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767836.337258:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767836.337260:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767836.337267:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775767836.337273:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767836.337278:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767836.337288:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767836.337290:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767836.337292:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767836.337349:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767836.337354:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767836.337356:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767836.337359:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767836.337362:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767836.337364:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767836.337366:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767836.337368:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767836.337371:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767836.401458:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767836.401462:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767836.401465:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775767837.361311:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767837.361321:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767837.361324:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767837.361329:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775767837.361334:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767837.361335:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767837.361341:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767837.361346:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767837.361347:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767837.361360:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00100000:3.0:1775767837.361360:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775767837.361366:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767837.361367:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767837.361371:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767837.361374:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767837.361378:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767837.361380:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767837.361383:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:3.0:1775767837.361500:0:2388:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.156:988 00000400:00000100:3.0:1775767837.361539:0:2388:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.156@tcp at host 192.168.202.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:3.0:1775767837.361548:0:2388:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767837.361551:0:2388:0:(router.c:1750:lnet_notify()) 192.168.202.56@tcp notifying 192.168.202.156@tcp: down 00000400:00000200:3.0:1775767837.361556:0:2388:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767837.361560:0:2388:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:3.0:1775767837.361565:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000800:00000010:3.0:1775767837.361569:0:2388:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6300. 00000400:00000200:3.0:1775767837.361574:0:2388:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1775767837.361579:0:2388:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1775767837.361586:0:2388:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014aa0 00000400:00000010:3.0:1775767837.361589:0:2388:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014aa0. 00000100:00000001:3.0:1775767837.361594:0:2388:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767837.361604:0:2388:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9942099aad80 x1862022294240640/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767859 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767837.361638:0:2388:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767837.361643:0:2388:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099aad80 x1862022294240640/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767859 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767837.361652:0:2388:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767837.361653:0:2388:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1775767837.361664:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000800:00000010:3.0:1775767837.361667:0:2388:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6500. 00000400:00000200:3.0:1775767837.361670:0:2388:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1775767837.361673:0:2388:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1775767837.361677:0:2388:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014ee0 00000400:00000010:3.0:1775767837.361679:0:2388:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014ee0. 00000100:00000001:3.0:1775767837.361681:0:2388:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767837.361686:0:2388:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9942099aa680 x1862022294240768/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767859 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767837.361695:0:2388:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767837.361698:0:2388:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099aa680 x1862022294240768/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767859 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767837.361704:0:2388:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767837.361706:0:2388:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1775767837.361711:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000800:00000010:3.0:1775767837.361713:0:2388:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6c00. 00000400:00000200:3.0:1775767837.361716:0:2388:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1775767837.361720:0:2388:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1775767837.361724:0:2388:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014220 00000400:00000010:3.0:1775767837.361726:0:2388:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014220. 00000100:00000001:3.0:1775767837.361728:0:2388:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767837.361732:0:2388:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9942099ab800 x1862022294240896/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767859 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767837.361740:0:2388:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767837.361743:0:2388:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ab800 x1862022294240896/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767859 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767837.361750:0:2388:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767837.361751:0:2388:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1775767837.361757:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000800:00000010:3.0:1775767837.361759:0:2388:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6600. 00000400:00000200:3.0:1775767837.361761:0:2388:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1775767837.361764:0:2388:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1775767837.361767:0:2388:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014a18 00000400:00000010:3.0:1775767837.361768:0:2388:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014a18. 00000100:00000001:3.0:1775767837.361771:0:2388:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767837.361775:0:2388:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9942099aa300 x1862022294241024/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767859 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767837.361794:0:2388:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767837.361797:0:2388:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099aa300 x1862022294241024/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767859 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767837.361803:0:2388:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767837.361805:0:2388:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1775767837.361810:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000800:00000010:3.0:1775767837.361812:0:2388:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6a00. 00000400:00000200:3.0:1775767837.361815:0:2388:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1775767837.361818:0:2388:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1775767837.361821:0:2388:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014330 00000400:00000010:3.0:1775767837.361823:0:2388:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014330. 00000100:00000001:3.0:1775767837.361825:0:2388:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767837.361829:0:2388:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9942099a8a80 x1862022294241152/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767859 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767837.361837:0:2388:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767837.361840:0:2388:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a8a80 x1862022294241152/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767859 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767837.361846:0:2388:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767837.361847:0:2388:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775767838.385290:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767838.385291:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767838.385302:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767838.385302:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767838.385307:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767838.385307:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767838.385320:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775767838.385321:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767838.385330:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767838.385332:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767838.385335:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767838.385338:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767838.385340:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767838.385345:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767838.385349:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767838.385349:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767838.385352:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767838.385355:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767838.449364:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767838.449368:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767838.449371:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775767839.409286:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767839.409292:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767839.409293:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767839.409298:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775767839.409304:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767839.409308:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767839.409310:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767839.409313:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767839.409315:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767839.409343:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:3.0:1775767839.409349:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775767839.409352:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767839.409360:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767839.409362:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000080:00000001:2.0:1775767839.409370:0:9316:0:(rw.c:1481:ll_writepages()) Process entered 00000100:00000001:3.0:1775767839.409370:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:1.0:1775767839.409378:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000080:00000001:2.0:1775767839.409378:0:9316:0:(file.c:5250:cl_sync_file_range()) Process entered 00000100:00080000:3.0:1775767839.409380:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775767839.409388:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1775767839.409389:0:9316:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00000001:3.0:1775767839.409394:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775767839.409395:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:2.0:1775767839.409399:0:9316:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994210b32630. 00000100:00000001:1.0:1775767839.409400:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767839.409403:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767839.409408:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1775767839.409408:0:9316:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9942060d4c00. 00000100:00080000:3.0:1775767839.409409:0:13471:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5091 00000020:00000010:2.0:1775767839.409419:0:9316:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff9942060d4800. 00000400:00000200:3.0:1775767839.409428:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000010:2.0:1775767839.409429:0:9316:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942182d6240. 00000008:00000010:2.0:1775767839.409439:0:9316:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207ccc2f0. 00000400:00000200:3.0:1775767839.409442:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00020000:00000010:2.0:1775767839.409465:0:9316:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994220156e40. 00000100:00080000:3.0:1775767839.409469:0:13471:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00008000:00000010:2.0:1775767839.409472:0:9316:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942077a8e20. 00000100:00000040:3.0:1775767839.409478:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000080:00000010:2.0:1775767839.409480:0:9316:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff9942087472a0. 00000100:00000001:3.0:1775767839.409483:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000080:00000010:2.0:1775767839.409489:0:9316:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b8000. 00000100:00000040:3.0:1775767839.409489:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767839.409496:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000010:2.0:1775767839.409498:0:9316:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9942060d5200. 00000020:00000001:3.0:1775767839.409504:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000008:00000010:2.0:1775767839.409506:0:9316:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942077cd600. 00000020:00000040:3.0:1775767839.409508:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00020000:00000010:2.0:1775767839.409512:0:9316:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff99420876d680. 00000020:00000001:3.0:1775767839.409514:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00008000:00000010:2.0:1775767839.409518:0:9316:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076cf60. 00000020:00000040:3.0:1775767839.409519:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:3.0:1775767839.409526:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000080:00000010:2.0:1775767839.409527:0:9316:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff994205066870. 00000020:00000001:3.0:1775767839.409530:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:2.0:1775767839.409535:0:9316:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107759908408 : -112965949643208 : ffff994210b32638) 00000100:00000040:3.0:1775767839.409538:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00001000:2.0:1775767839.409542:0:9316:0:(cl_object.c:831:cl_env_get()) 1@ffff994210b32630 00000100:00000001:3.0:1775767839.409544:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767839.409550:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000001:2.0:1775767839.409552:0:9316:0:(cl_object.c:151:cl_object_top()) ffff99420791a5f8 -> ffff99420791a5f8 00000100:00000001:3.0:1775767839.409555:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775767839.409557:0:9316:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000040:3.0:1775767839.409564:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000080:00000001:2.0:1775767839.409565:0:9316:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:2.0:1775767839.409572:0:9316:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x10176:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000100:00080000:3.0:1775767839.409575:0:13471:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767839.409579:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767839.409584:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000020:00000001:2.0:1775767839.409586:0:9316:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775767839.409590:0:9316:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 02000000:00000001:3.0:1775767839.409591:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000001:2.0:1775767839.409593:0:9316:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767839.409599:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775767839.409601:0:9316:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x10176:0x0]io ffff9942077b8060 type 6 ignore/verify layout 1/0 00000100:00000001:3.0:1775767839.409606:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00020000:00000002:2.0:1775767839.409609:0:9316:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207dfb6c8) owner 0000000000000000/ffff994201384f80 00000100:00000001:3.0:1775767839.409611:0:13471:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767839.409614:0:9316:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:2.0:1775767839.409618:0:9316:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:2.0:1775767839.409620:0:9316:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00020000:00000001:2.0:1775767839.409622:0:9316:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767839.409625:0:9316:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000010:3.0:1775767839.409627:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aad80. 00020000:00000001:2.0:1775767839.409630:0:9316:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:2.0:1775767839.409633:0:9316:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767839.409635:0:9316:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000040:3.0:1775767839.409635:0:13471:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 00000020:00000001:2.0:1775767839.409641:0:9316:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:2.0:1775767839.409644:0:9316:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 02000000:00000001:3.0:1775767839.409646:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00020000:00000002:2.0:1775767839.409647:0:9316:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207dfb6c8) owner 0000000000000000/ffff994201384f80 02000000:00000001:3.0:1775767839.409652:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767839.409654:0:9316:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767839.409659:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000020:00000001:2.0:1775767839.409662:0:9316:0:(cl_io.c:823:cl_io_loop()) Process entered 02000000:00000001:3.0:1775767839.409665:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000020:00000001:2.0:1775767839.409668:0:9316:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00020000:00000001:2.0:1775767839.409672:0:9316:0:(lov_io.c:836:lov_io_iter_init()) Process entered 02000000:00000001:3.0:1775767839.409675:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00020000:00200000:2.0:1775767839.409678:0:9316:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00000002:2.0:1775767839.409686:0:9316:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:2.0:1775767839.409690:0:9316:0:(lov_io.c:155:lov_sub_get()) Process entered 00020000:00000001:2.0:1775767839.409692:0:9316:0:(lov_io.c:104:lov_io_sub_init()) Process entered 02000000:00000010:3.0:1775767839.409692:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0c00. 00000020:00000001:2.0:1775767839.409697:0:9316:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775767839.409703:0:9316:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994210b32090. 00000100:00000001:3.0:1775767839.409707:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1775767839.409709:0:9316:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9942060d5800. 00000020:00000010:2.0:1775767839.409716:0:9316:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff9942060d5a00. 00000020:00000010:2.0:1775767839.409719:0:9316:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942182d64e0. 00000008:00000010:2.0:1775767839.409724:0:9316:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207cc8b28. 00000100:00100000:3.0:1775767839.409726:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aad80 x1862022294241280/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00020000:00000010:2.0:1775767839.409730:0:9316:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994220156840. 00008000:00000010:2.0:1775767839.409739:0:9316:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942077a9530. 00000080:00000010:2.0:1775767839.409750:0:9316:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994208747738. 00000100:00000001:3.0:1775767839.409755:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000080:00000010:2.0:1775767839.409756:0:9316:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077ba910. 00000020:00000010:2.0:1775767839.409765:0:9316:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994209d2dc00. 00000100:00000001:3.0:1775767839.409767:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:2.0:1775767839.409772:0:9316:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942077cd8c0. 00020000:00000010:2.0:1775767839.409781:0:9316:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff99420876e400. 00000100:00000040:3.0:1775767839.409781:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000045ccf08] to pc [ptlrpcd_rcv+-1] req@ffff9942099aad80 x1862022294241280/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00008000:00000010:2.0:1775767839.409786:0:9316:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076c3f0. 00000080:00000010:2.0:1775767839.409795:0:9316:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942050663c0. 00000020:00000001:2.0:1775767839.409808:0:9316:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107759906968 : -112965949644648 : ffff994210b32098) 00000020:00001000:2.0:1775767839.409818:0:9316:0:(cl_object.c:831:cl_env_get()) 1@ffff994210b32090 00000020:00000001:2.0:1775767839.409822:0:9316:0:(cl_object.c:151:cl_object_top()) ffff99420791ae90 -> ffff99420791a5f8 00000020:00000001:2.0:1775767839.409824:0:9316:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:2.0:1775767839.409827:0:9316:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775767839.409829:0:9316:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:2.0:1775767839.409830:0:9316:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767839.409832:0:9316:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767839.409835:0:9316:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107621738216 : -112966087813400 : ffff99420876d6e8) 00000020:00000001:2.0:1775767839.409838:0:9316:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000020:00000001:2.0:1775767839.409839:0:9316:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767839.409842:0:9316:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767839.409844:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767839.409846:0:9316:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767839.409851:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000001:2.0:1775767839.409852:0:9316:0:(cl_io.c:314:cl_io_lock()) Process entered 00020000:00000001:2.0:1775767839.409859:0:9316:0:(lov_io.c:1108:lov_io_lock()) Process entered 00000100:00080000:3.0:1775767839.409861:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00020000:00000001:2.0:1775767839.409863:0:9316:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000001:3.0:1775767839.409865:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000020:00000001:2.0:1775767839.409866:0:9316:0:(cl_io.c:314:cl_io_lock()) Process entered 00000100:00080000:3.0:1775767839.409871:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000020:00000001:2.0:1775767839.409872:0:9316:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000100:00000001:3.0:1775767839.409879:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767839.409883:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5091 00000400:00000200:3.0:1775767839.409889:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:2.0:1775767839.409892:0:9316:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767839.409897:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000020:00000001:2.0:1775767839.409902:0:9316:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767839.409903:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767839.409911:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00020000:00000001:2.0:1775767839.409913:0:9316:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767839.409917:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00020000:00000001:2.0:1775767839.409919:0:9316:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767839.409924:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000001:2.0:1775767839.409925:0:9316:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:2.0:1775767839.409930:0:9316:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767839.409931:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775767839.409934:0:9316:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767839.409939:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000001:2.0:1775767839.409941:0:9316:0:(cl_io.c:528:cl_io_start()) Process entered 00000020:00000040:3.0:1775767839.409942:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00020000:00000001:2.0:1775767839.409946:0:9316:0:(lov_io.c:1114:lov_io_start()) Process entered 00000020:00000001:3.0:1775767839.409947:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00020000:00000001:2.0:1775767839.409950:0:9316:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000040:3.0:1775767839.409952:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:2.0:1775767839.409954:0:9316:0:(cl_io.c:528:cl_io_start()) Process entered 00000020:00000001:3.0:1775767839.409959:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000008:00000001:2.0:1775767839.409961:0:9316:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000020:00000001:3.0:1775767839.409966:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000008:00000001:2.0:1775767839.409968:0:9316:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000100:00000040:3.0:1775767839.409975:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000008:00000001:2.0:1775767839.409976:0:9316:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000100:00000001:3.0:1775767839.409979:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000008:00000020:2.0:1775767839.409981:0:9316:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000100:00000040:3.0:1775767839.409985:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000008:00000001:2.0:1775767839.409987:0:9316:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767839.409992:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000008:00000001:2.0:1775767839.409994:0:9316:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000008:00000001:2.0:1775767839.409997:0:9316:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000020:00000040:3.0:1775767839.409998:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000008:00000001:2.0:1775767839.410002:0:9316:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107754940544 : -112965954611072 : ffff994210675880) 00000100:00080000:3.0:1775767839.410006:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000008:00000020:2.0:1775767839.410012:0:9316:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994210675880 ready 0|-|- wr 1|-|+ rd 0|- 10 in flight 00000100:00000001:3.0:1775767839.410014:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767839.410021:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000008:00000001:2.0:1775767839.410028:0:9316:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:2.0:1775767839.410031:0:9316:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:2.0:1775767839.410032:0:9316:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1775767839.410034:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00000001:2.0:1775767839.410036:0:9316:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 02000000:00000001:3.0:1775767839.410041:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000008:00000020:2.0:1775767839.410042:0:9316:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994218232fc0 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000100:00000001:3.0:1775767839.410047:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00000001:2.0:1775767839.410051:0:9316:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1775767839.410056:0:9316:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767839.410058:0:9316:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767839.410060:0:9316:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767839.410062:0:9316:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767839.410064:0:9316:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767839.410065:0:9316:0:(cl_io.c:555:cl_io_end()) Process entered 00020000:00000001:2.0:1775767839.410067:0:9316:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00020000:00000001:2.0:1775767839.410069:0:9316:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000020:00000001:2.0:1775767839.410070:0:9316:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000001:2.0:1775767839.410071:0:9316:0:(cl_io.c:563:cl_io_end()) Process leaving 00020000:00000001:2.0:1775767839.410073:0:9316:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767839.410075:0:9316:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000020:00000001:2.0:1775767839.410076:0:9316:0:(cl_io.c:563:cl_io_end()) Process leaving 00000020:00000001:2.0:1775767839.410077:0:9316:0:(cl_io.c:352:cl_io_unlock()) Process entered 00020000:00000001:2.0:1775767839.410080:0:9316:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00020000:00000001:2.0:1775767839.410081:0:9316:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775767839.410082:0:9316:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000020:00000001:2.0:1775767839.410083:0:9316:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:2.0:1775767839.410084:0:9316:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767839.410086:0:9316:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767839.410088:0:9316:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000020:00000001:2.0:1775767839.410089:0:9316:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:2.0:1775767839.410091:0:9316:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000020:00000001:2.0:1775767839.410092:0:9316:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000001:2.0:1775767839.410093:0:9316:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:2.0:1775767839.410095:0:9316:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00020000:00000001:2.0:1775767839.410096:0:9316:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775767839.410097:0:9316:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000001:2.0:1775767839.410099:0:9316:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00020000:00000001:2.0:1775767839.410101:0:9316:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767839.410103:0:9316:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767839.410105:0:9316:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000020:00000001:2.0:1775767839.410106:0:9316:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000020:00000001:2.0:1775767839.410107:0:9316:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767839.410110:0:9316:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:2.0:1775767839.410111:0:9316:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:2.0:1775767839.410114:0:9316:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:2.0:1775767839.410115:0:9316:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:2.0:1775767839.410117:0:9316:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:2.0:1775767839.410119:0:9316:0:(cl_object.c:925:cl_env_put()) 1@ffff994210b32090 00000020:00000010:2.0:1775767839.410123:0:9316:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff9942060d5a00. 00000020:00000010:2.0:1775767839.410131:0:9316:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942182d64e0. 00000008:00000010:2.0:1775767839.410136:0:9316:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207cc8b28. 00020000:00000010:2.0:1775767839.410141:0:9316:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994220156840. 00008000:00000010:2.0:1775767839.410143:0:9316:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942077a9530. 00000080:00000010:2.0:1775767839.410147:0:9316:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994208747738. 00000080:00000010:2.0:1775767839.410151:0:9316:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077ba910. 00000008:00100000:3.0:1775767839.410209:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000020:00000010:2.0:1775767839.410213:0:9316:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9942060d5800. 00000008:00000001:3.0:1775767839.410216:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767839.410222:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:2.0:1775767839.410224:0:9316:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942077cd8c0. 00000100:00000010:3.0:1775767839.410229:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aa680. 00020000:00000010:2.0:1775767839.410230:0:9316:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff99420876e400. 00000020:00000040:3.0:1775767839.410238:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00008000:00000010:2.0:1775767839.410241:0:9316:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076c3f0. 02000000:00000001:3.0:1775767839.410244:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767839.410248:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:2.0:1775767839.410249:0:9316:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942050663c0. 00000100:00000001:3.0:1775767839.410251:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767839.410257:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767839.410262:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1775767839.410264:0:9316:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994209d2dc00. 02000000:00000010:3.0:1775767839.410268:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2800. 00000020:00000010:2.0:1775767839.410271:0:9316:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994210b32090. 00000100:00000001:3.0:1775767839.410277:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767839.410279:0:9316:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:2.0:1775767839.410288:0:9316:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000100:00100000:3.0:1775767839.410292:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aa680 x1862022294241408/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000080:00000001:2.0:1775767839.410293:0:9316:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000080:00200000:2.0:1775767839.410299:0:9316:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x10176:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:2.0:1775767839.410321:0:9316:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:2.0:1775767839.410325:0:9316:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:2.0:1775767839.410329:0:9316:0:(cl_object.c:925:cl_env_put()) 1@ffff994210b32630 00000020:00000010:2.0:1775767839.410332:0:9316:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff9942060d4800. 00000020:00000010:2.0:1775767839.410334:0:9316:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942182d6240. 00000008:00000010:2.0:1775767839.410337:0:9316:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207ccc2f0. 00020000:00000010:2.0:1775767839.410340:0:9316:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994220156e40. 00000100:00000001:3.0:1775767839.410342:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00008000:00000010:2.0:1775767839.410345:0:9316:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942077a8e20. 00000100:00000001:3.0:1775767839.410349:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:2.0:1775767839.410352:0:9316:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff9942087472a0. 00000080:00000010:2.0:1775767839.410358:0:9316:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b8000. 00000100:00000040:3.0:1775767839.410361:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006cb1597d] to pc [ptlrpcd_rcv+-1] req@ffff9942099aa680 x1862022294241408/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1775767839.410364:0:9316:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9942060d4c00. 00000008:00000010:2.0:1775767839.410376:0:9316:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942077cd600. 00020000:00000010:2.0:1775767839.410391:0:9316:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff99420876d680. 00008000:00000010:2.0:1775767839.410401:0:9316:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076cf60. 00000100:00000001:3.0:1775767839.410403:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:2.0:1775767839.410409:0:9316:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff994205066870. 00000100:00080000:3.0:1775767839.410410:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000010:2.0:1775767839.410422:0:9316:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9942060d5200. 00000100:00080000:3.0:1775767839.410423:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775767839.410430:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000020:00000010:2.0:1775767839.410433:0:9316:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994210b32630. 00000100:00080000:3.0:1775767839.410433:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767839.410460:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000080:00000001:2.0:1775767839.410464:0:9316:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775767839.410465:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5091 00000080:00000001:2.0:1775767839.410476:0:9316:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767839.410478:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767839.410490:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767839.410493:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767839.410498:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767839.410502:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767839.410506:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767839.410509:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767839.410514:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767839.410517:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:3.0:1775767839.410519:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767839.410521:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:3.0:1775767839.410524:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:3.0:1775767839.410527:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:3.0:1775767839.410532:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767839.410535:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767839.410538:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767839.410540:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767839.410543:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:3.0:1775767839.410546:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767839.410548:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767839.410550:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767839.410555:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767839.410557:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767839.410560:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767839.410568:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775767839.410570:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767839.410572:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767839.410577:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ab800. 00000020:00000040:3.0:1775767839.410579:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:3.0:1775767839.410582:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767839.410583:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767839.410585:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767839.410587:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767839.410589:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767839.410594:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0000. 00000100:00000001:3.0:1775767839.410597:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767839.410603:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ab800 x1862022294241536/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767839.410618:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767839.410621:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767839.410625:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000089df80c0] to pc [ptlrpcd_rcv+-1] req@ffff9942099ab800 x1862022294241536/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767839.410634:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767839.410637:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767839.410641:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775767839.410643:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767839.410644:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767839.410647:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767839.410650:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5091 00000400:00000200:3.0:1775767839.410654:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767839.410657:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767839.410660:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767839.410662:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767839.410665:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767839.410667:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767839.410669:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767839.410671:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767839.410672:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:3.0:1775767839.410674:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767839.410676:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:3.0:1775767839.410677:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:3.0:1775767839.410679:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:3.0:1775767839.410682:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767839.410684:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767839.410687:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767839.410689:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767839.410692:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:3.0:1775767839.410695:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767839.410698:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767839.410700:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767839.410703:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767839.410705:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767839.410707:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767839.410709:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:3.0:1775767839.410712:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767839.410714:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767839.410718:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aa300. 00000020:00000040:3.0:1775767839.410720:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:3.0:1775767839.410723:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767839.410724:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767839.410726:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767839.410727:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767839.410728:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767839.410733:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2000. 00000100:00000001:3.0:1775767839.410735:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767839.410740:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aa300 x1862022294241664/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767839.410747:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767839.410749:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767839.410752:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f33bdf15] to pc [ptlrpcd_rcv+-1] req@ffff9942099aa300 x1862022294241664/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767839.410760:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767839.410763:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767839.410767:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775767839.410769:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767839.410770:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767839.410773:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767839.410777:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5091 00000400:00000200:3.0:1775767839.410781:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767839.410784:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767839.410786:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767839.410789:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767839.410790:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767839.410792:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767839.410794:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767839.410797:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767839.410798:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:3.0:1775767839.410799:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767839.410801:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:3.0:1775767839.410802:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:3.0:1775767839.410804:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:3.0:1775767839.410807:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767839.410809:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767839.410811:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767839.410813:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767839.410815:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:3.0:1775767839.410817:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767839.410819:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767839.410820:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767839.410822:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767839.410824:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767839.410826:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767839.410828:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:3.0:1775767839.410830:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767839.410832:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767839.410836:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8a80. 00000020:00000040:3.0:1775767839.410837:0:13471:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:3.0:1775767839.410840:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767839.410841:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767839.410843:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767839.410845:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767839.410846:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767839.410850:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0400. 00000100:00000001:3.0:1775767839.410854:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767839.410857:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8a80 x1862022294241792/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767839.410865:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767839.410867:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767839.410871:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006c3265af] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8a80 x1862022294241792/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767839.410879:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767839.410884:0:13471:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5101) 00000800:00000200:0.1:1775767839.411264:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767839.411268:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775767839.411706:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767839.411709:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775767839.412033:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767839.412036:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775767839.412387:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767839.412390:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775767839.412741:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767839.412744:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000100:00000001:2.0:1775767840.433308:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767840.433321:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767840.433327:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767840.433336:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775767840.433362:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767840.433367:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767840.433380:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767840.433384:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767840.433387:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767840.433459:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767840.433463:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767840.433464:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767840.433467:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775767840.433470:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767840.433472:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767840.433473:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767840.433475:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767840.433477:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767840.497392:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767840.497396:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767840.497399:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775767840.753245:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299765567, at 4299765568 00000100:00000010:1.0:1775767840.753271:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a39c0. 00000100:00000001:3.0:1775767841.457364:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767841.457372:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767841.457374:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767841.457374:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767841.457382:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767841.457387:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767841.457390:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:1.0:1775767841.457392:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767841.457396:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767841.457398:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767841.457403:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767841.457404:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767841.457409:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767841.457413:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767841.457416:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767841.457417:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767841.457419:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767841.457422:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767842.481236:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767842.481243:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767842.481246:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767842.481252:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767842.481257:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767842.481261:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767842.481263:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767842.481265:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767842.481269:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767842.481298:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767842.481303:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767842.481305:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767842.481309:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767842.481313:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767842.481333:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767842.481345:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767842.481348:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767842.481351:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767842.545301:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767842.545304:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767842.545307:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775767843.505250:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767843.505259:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767843.505262:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767843.505269:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767843.505277:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767843.505280:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767843.505285:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767843.505286:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767843.505292:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767843.505294:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00100000:3.0:1775767843.505297:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767843.505298:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767843.505305:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767843.505311:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767843.505312:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767843.505322:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767843.505326:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767843.505329:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767844.529271:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767844.529275:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767844.529276:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767844.529279:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767844.529290:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000080:00000001:2.0:1775767844.529295:0:9316:0:(rw.c:1481:ll_writepages()) Process entered 00000100:00000001:3.0:1775767844.529295:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767844.529296:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000080:00000001:2.0:1775767844.529299:0:9316:0:(file.c:5250:cl_sync_file_range()) Process entered 00000100:00000001:1.0:1775767844.529302:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1775767844.529303:0:9316:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00100000:3.0:1775767844.529303:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767844.529306:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767844.529309:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767844.529309:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767844.529314:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767844.529315:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767844.529320:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767844.529321:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1775767844.529321:0:9316:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994210b32630. 00000100:00000001:3.0:1775767844.529323:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767844.529329:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1775767844.529332:0:9316:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994209d2dc00. 00000020:00000010:2.0:1775767844.529339:0:9316:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994209d2d600. 00000020:00000010:2.0:1775767844.529345:0:9316:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942182d6240. 00000008:00000010:2.0:1775767844.529363:0:9316:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207ccc2f0. 00020000:00000010:2.0:1775767844.529369:0:9316:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994220156e40. 00000100:00080000:3.0:1775767844.529373:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00008000:00000010:2.0:1775767844.529375:0:9316:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942077a8e20. 00000100:00080000:3.0:1775767844.529378:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775767844.529382:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000080:00000010:2.0:1775767844.529383:0:9316:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff9942087472a0. 00000100:00080000:3.0:1775767844.529385:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767844.529389:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000080:00000010:2.0:1775767844.529390:0:9316:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b8000. 00000100:00080000:3.0:1775767844.529396:0:13471:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5096 00000020:00000010:2.0:1775767844.529409:0:9316:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994209d2cc00. 00000008:00000010:2.0:1775767844.529415:0:9316:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942077cd600. 00020000:00000010:2.0:1775767844.529420:0:9316:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff99420876d680. 00000400:00000200:3.0:1775767844.529421:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00008000:00000010:2.0:1775767844.529427:0:9316:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076cf60. 00000400:00000200:3.0:1775767844.529429:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767844.529433:0:13471:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000080:00000010:2.0:1775767844.529434:0:9316:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff994205066870. 00000100:00000040:3.0:1775767844.529436:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000001:2.0:1775767844.529440:0:9316:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107759908408 : -112965949643208 : ffff994210b32638) 00000100:00000001:3.0:1775767844.529440:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767844.529442:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00001000:2.0:1775767844.529448:0:9316:0:(cl_object.c:831:cl_env_get()) 1@ffff994210b32630 00000100:00000001:3.0:1775767844.529448:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767844.529453:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767844.529454:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:2.0:1775767844.529455:0:9316:0:(cl_object.c:151:cl_object_top()) ffff9942078a76c0 -> ffff9942078a76c0 00000020:00000001:3.0:1775767844.529457:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767844.529460:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:2.0:1775767844.529461:0:9316:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:3.0:1775767844.529464:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000080:00000001:2.0:1775767844.529466:0:9316:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000020:00000001:3.0:1775767844.529466:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000080:00200000:2.0:1775767844.529470:0:9316:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000100:00000040:3.0:1775767844.529471:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767844.529474:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767844.529477:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767844.529478:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775767844.529479:0:9316:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000040:3.0:1775767844.529481:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000020:00000001:2.0:1775767844.529482:0:9316:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00080000:3.0:1775767844.529485:0:13471:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000080:00000001:2.0:1775767844.529486:0:9316:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767844.529487:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767844.529491:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00020000:00000002:2.0:1775767844.529493:0:9316:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017a:0x0]io ffff9942077b8060 type 6 ignore/verify layout 1/0 02000000:00000001:3.0:1775767844.529494:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000002:2.0:1775767844.529500:0:9316:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207c1f9b0) owner 0000000000000000/ffff994201384f80 02000000:00000001:3.0:1775767844.529500:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767844.529504:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00020000:00000001:2.0:1775767844.529505:0:9316:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:2.0:1775767844.529508:0:9316:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00000100:00000001:3.0:1775767844.529508:0:13471:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767844.529511:0:9316:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00020000:00000001:2.0:1775767844.529513:0:9316:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767844.529514:0:9316:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:2.0:1775767844.529516:0:9316:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00000100:00000010:3.0:1775767844.529516:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8e00. 00020000:00000001:2.0:1775767844.529519:0:9316:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767844.529521:0:9316:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000040:3.0:1775767844.529521:0:13471:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 00000020:00000001:2.0:1775767844.529522:0:9316:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:2.0:1775767844.529524:0:9316:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00020000:00000002:2.0:1775767844.529527:0:9316:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207c1f9b0) owner 0000000000000000/ffff994201384f80 00000020:00000001:2.0:1775767844.529529:0:9316:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767844.529529:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767844.529532:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767844.529533:0:9316:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:2.0:1775767844.529535:0:9316:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000100:00000001:3.0:1775767844.529537:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00020000:00000001:2.0:1775767844.529538:0:9316:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00020000:00200000:2.0:1775767844.529541:0:9316:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 02000000:00000001:3.0:1775767844.529542:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00020000:00000002:2.0:1775767844.529546:0:9316:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 02000000:00000001:3.0:1775767844.529548:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767844.529549:0:9316:0:(lov_io.c:155:lov_sub_get()) Process entered 00020000:00000001:2.0:1775767844.529552:0:9316:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000020:00000001:2.0:1775767844.529553:0:9316:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775767844.529556:0:9316:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994210b32090. 02000000:00000010:3.0:1775767844.529558:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b1000. 00000020:00000010:2.0:1775767844.529559:0:9316:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994209d2de00. 00000020:00000010:2.0:1775767844.529563:0:9316:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994209d2d000. 00000020:00000010:2.0:1775767844.529565:0:9316:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942182d64e0. 00000100:00000001:3.0:1775767844.529567:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:2.0:1775767844.529568:0:9316:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207cc8b28. 00020000:00000010:2.0:1775767844.529572:0:9316:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994220156840. 00008000:00000010:2.0:1775767844.529574:0:9316:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942077a9530. 00000100:00100000:3.0:1775767844.529577:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8e00 x1862022294241920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000080:00000010:2.0:1775767844.529578:0:9316:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994208747738. 00000080:00000010:2.0:1775767844.529583:0:9316:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077ba910. 00000020:00000010:2.0:1775767844.529589:0:9316:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9942060d5200. 00000100:00000001:3.0:1775767844.529590:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000008:00000010:2.0:1775767844.529592:0:9316:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942077cd8c0. 00000100:00000001:3.0:1775767844.529595:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:2.0:1775767844.529596:0:9316:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff99420876e400. 00008000:00000010:2.0:1775767844.529599:0:9316:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076c3f0. 00000100:00000040:3.0:1775767844.529600:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000864a6866] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8e00 x1862022294241920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000010:2.0:1775767844.529603:0:9316:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942050663c0. 00000020:00000001:2.0:1775767844.529607:0:9316:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107759906968 : -112965949644648 : ffff994210b32098) 00000020:00001000:2.0:1775767844.529612:0:9316:0:(cl_object.c:831:cl_env_get()) 1@ffff994210b32090 00000020:00000001:2.0:1775767844.529616:0:9316:0:(cl_object.c:151:cl_object_top()) ffff9942078a7788 -> ffff9942078a76c0 00000020:00000001:2.0:1775767844.529618:0:9316:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:2.0:1775767844.529621:0:9316:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775767844.529622:0:9316:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:2.0:1775767844.529623:0:9316:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767844.529624:0:9316:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767844.529625:0:9316:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107621738216 : -112966087813400 : ffff99420876d6e8) 00000100:00000001:3.0:1775767844.529627:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767844.529628:0:9316:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000020:00000001:2.0:1775767844.529631:0:9316:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767844.529632:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00020000:00000001:2.0:1775767844.529633:0:9316:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767844.529637:0:9316:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767844.529637:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000020:00000001:2.0:1775767844.529639:0:9316:0:(cl_io.c:314:cl_io_lock()) Process entered 00000100:00000001:3.0:1775767844.529640:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00020000:00000001:2.0:1775767844.529641:0:9316:0:(lov_io.c:1108:lov_io_lock()) Process entered 00000100:00080000:3.0:1775767844.529643:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00020000:00000001:2.0:1775767844.529644:0:9316:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775767844.529646:0:9316:0:(cl_io.c:314:cl_io_lock()) Process entered 00000100:00000001:3.0:1775767844.529647:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000020:00000001:2.0:1775767844.529648:0:9316:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:2.0:1775767844.529650:0:9316:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767844.529651:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5096 00000020:00000001:2.0:1775767844.529654:0:9316:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767844.529657:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:2.0:1775767844.529670:0:9316:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767844.529671:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00020000:00000001:2.0:1775767844.529672:0:9316:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767844.529674:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000020:00000001:2.0:1775767844.529675:0:9316:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:2.0:1775767844.529678:0:9316:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767844.529680:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000001:2.0:1775767844.529681:0:9316:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767844.529684:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:2.0:1775767844.529685:0:9316:0:(cl_io.c:528:cl_io_start()) Process entered 00020000:00000001:2.0:1775767844.529688:0:9316:0:(lov_io.c:1114:lov_io_start()) Process entered 00000100:00000040:3.0:1775767844.529689:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00020000:00000001:2.0:1775767844.529690:0:9316:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000001:3.0:1775767844.529692:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775767844.529693:0:9316:0:(cl_io.c:528:cl_io_start()) Process entered 00000008:00000001:2.0:1775767844.529696:0:9316:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000020:00000001:3.0:1775767844.529697:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000008:00000001:2.0:1775767844.529699:0:9316:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000020:00000040:3.0:1775767844.529703:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000008:00000001:2.0:1775767844.529704:0:9316:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:2.0:1775767844.529707:0:9316:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000020:00000001:3.0:1775767844.529708:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000008:00000001:2.0:1775767844.529709:0:9316:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1775767844.529712:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:3.0:1775767844.529716:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:3.0:1775767844.529718:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000008:00000001:2.0:1775767844.529732:0:9316:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000100:00000040:3.0:1775767844.529735:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000008:00000001:2.0:1775767844.529736:0:9316:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000008:00000001:2.0:1775767844.529740:0:9316:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107884691392 : -112965824860224 : ffff994218232fc0) 00000100:00000001:3.0:1775767844.529741:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000008:00000020:2.0:1775767844.529745:0:9316:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994218232fc0 ready 0|-|- wr 1|-|+ rd 0|- 10 in flight 00000100:00000040:3.0:1775767844.529746:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767844.529751:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000008:00000001:2.0:1775767844.529753:0:9316:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:2.0:1775767844.529756:0:9316:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000020:00000040:3.0:1775767844.529756:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000008:00000001:2.0:1775767844.529759:0:9316:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1775767844.529762:0:9316:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000100:00080000:3.0:1775767844.529762:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000008:00000020:2.0:1775767844.529766:0:9316:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994210675880 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000100:00000001:3.0:1775767844.529768:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1775767844.529774:0:9316:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1775767844.529774:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000008:00000001:2.0:1775767844.529778:0:9316:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767844.529780:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1775767844.529782:0:9316:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767844.529787:0:9316:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767844.529787:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767844.529788:0:9316:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767844.529791:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000020:00000001:2.0:1775767844.529792:0:9316:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767844.529794:0:9316:0:(cl_io.c:555:cl_io_end()) Process entered 00000008:00100000:3.0:1775767844.529796:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00020000:00000001:2.0:1775767844.529797:0:9316:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00020000:00000001:2.0:1775767844.529799:0:9316:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000008:00000001:3.0:1775767844.529801:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767844.529802:0:9316:0:(cl_io.c:555:cl_io_end()) Process entered 00000100:00000001:3.0:1775767844.529803:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767844.529805:0:9316:0:(cl_io.c:563:cl_io_end()) Process leaving 00020000:00000001:2.0:1775767844.529808:0:9316:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767844.529809:0:9316:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000100:00000010:3.0:1775767844.529810:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ab480. 00000020:00000001:2.0:1775767844.529812:0:9316:0:(cl_io.c:563:cl_io_end()) Process leaving 00000020:00000040:3.0:1775767844.529815:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:2.0:1775767844.529816:0:9316:0:(cl_io.c:352:cl_io_unlock()) Process entered 02000000:00000001:3.0:1775767844.529828:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00020000:00000001:2.0:1775767844.529829:0:9316:0:(lov_io.c:1208:lov_io_unlock()) Process entered 02000000:00000001:3.0:1775767844.529829:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767844.529831:0:9316:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775767844.529834:0:9316:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000100:00000001:3.0:1775767844.529834:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000020:00000001:2.0:1775767844.529838:0:9316:0:(cl_io.c:373:cl_io_unlock()) Process leaving 02000000:00000001:3.0:1775767844.529838:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00020000:00000001:2.0:1775767844.529840:0:9316:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767844.529841:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767844.529846:0:9316:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767844.529848:0:9316:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 02000000:00000010:3.0:1775767844.529849:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b3400. 00000020:00000001:2.0:1775767844.529850:0:9316:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:2.0:1775767844.529853:0:9316:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000020:00000001:2.0:1775767844.529855:0:9316:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000100:00000001:3.0:1775767844.529855:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767844.529856:0:9316:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:2.0:1775767844.529860:0:9316:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00020000:00000001:2.0:1775767844.529863:0:9316:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00100000:3.0:1775767844.529864:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ab480 x1862022294242048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000001:2.0:1775767844.529865:0:9316:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000001:2.0:1775767844.529868:0:9316:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00020000:00000001:2.0:1775767844.529870:0:9316:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767844.529875:0:9316:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767844.529880:0:9316:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000020:00000001:2.0:1775767844.529883:0:9316:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000100:00000001:3.0:1775767844.529884:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000020:00000001:2.0:1775767844.529886:0:9316:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767844.529888:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767844.529890:0:9316:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:2.0:1775767844.529894:0:9316:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:2.0:1775767844.529896:0:9316:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000100:00000040:3.0:1775767844.529897:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000067a0ba74] to pc [ptlrpcd_rcv+-1] req@ffff9942099ab480 x1862022294242048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1775767844.529899:0:9316:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:2.0:1775767844.529902:0:9316:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:2.0:1775767844.529905:0:9316:0:(cl_object.c:925:cl_env_put()) 1@ffff994210b32090 00000020:00000010:2.0:1775767844.529912:0:9316:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994209d2d000. 00000100:00000001:3.0:1775767844.529916:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767844.529918:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000010:2.0:1775767844.529921:0:9316:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942182d64e0. 00000100:00080000:3.0:1775767844.529923:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000008:00000010:2.0:1775767844.529926:0:9316:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207cc8b28. 00000100:00000001:3.0:1775767844.529927:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767844.529930:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00020000:00000010:2.0:1775767844.529931:0:9316:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994220156840. 00000100:00000001:3.0:1775767844.529933:0:13471:0:(import.c:484:import_select_connection()) Process entered 00008000:00000010:2.0:1775767844.529936:0:9316:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942077a9530. 00000100:00080000:3.0:1775767844.529937:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5096 00000080:00000010:2.0:1775767844.529943:0:9316:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994208747738. 00000400:00000200:3.0:1775767844.529946:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000080:00000010:2.0:1775767844.529950:0:9316:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077ba910. 00000400:00000200:3.0:1775767844.529952:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767844.529957:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000020:00000010:2.0:1775767844.529958:0:9316:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994209d2de00. 00000100:00000040:3.0:1775767844.529962:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000008:00000010:2.0:1775767844.529965:0:9316:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942077cd8c0. 00000100:00000001:3.0:1775767844.529965:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767844.529971:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00020000:00000010:2.0:1775767844.529973:0:9316:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff99420876e400. 00000100:00000001:3.0:1775767844.529975:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00008000:00000010:2.0:1775767844.529978:0:9316:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076c3f0. 00000020:00000001:3.0:1775767844.529979:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000080:00000010:2.0:1775767844.529983:0:9316:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942050663c0. 00000020:00000040:3.0:1775767844.529983:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:3.0:1775767844.529986:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000010:2.0:1775767844.529988:0:9316:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9942060d5200. 00000020:00000040:3.0:1775767844.529989:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000010:2.0:1775767844.529993:0:9316:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994210b32090. 00000020:00000001:3.0:1775767844.529994:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:3.0:1775767844.529999:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00020000:00000001:2.0:1775767844.530000:0:9316:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:2.0:1775767844.530005:0:9316:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000100:00000040:3.0:1775767844.530005:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000080:00000001:2.0:1775767844.530008:0:9316:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000100:00000001:3.0:1775767844.530011:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000080:00200000:2.0:1775767844.530013:0:9316:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00000040:3.0:1775767844.530017:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000080:00000001:2.0:1775767844.530018:0:9316:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:2.0:1775767844.530021:0:9316:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000100:00000001:3.0:1775767844.530022:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00001000:2.0:1775767844.530025:0:9316:0:(cl_object.c:925:cl_env_put()) 1@ffff994210b32630 00000020:00000040:3.0:1775767844.530027:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000020:00000010:2.0:1775767844.530028:0:9316:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994209d2d600. 00000020:00000010:2.0:1775767844.530031:0:9316:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942182d6240. 00000100:00080000:3.0:1775767844.530033:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000008:00000010:2.0:1775767844.530037:0:9316:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207ccc2f0. 00000100:00000001:3.0:1775767844.530038:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767844.530041:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767844.530043:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767844.530044:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767844.530048:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00020000:00000010:2.0:1775767844.530049:0:9316:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994220156e40. 00000008:00100000:3.0:1775767844.530050:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00008000:00000010:2.0:1775767844.530053:0:9316:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942077a8e20. 00000008:00000001:3.0:1775767844.530053:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767844.530058:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:2.0:1775767844.530059:0:9316:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff9942087472a0. 00000100:00000010:3.0:1775767844.530061:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a9180. 00000080:00000010:2.0:1775767844.530062:0:9316:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b8000. 00000020:00000040:3.0:1775767844.530065:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000010:2.0:1775767844.530066:0:9316:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994209d2dc00. 02000000:00000001:3.0:1775767844.530071:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000008:00000010:2.0:1775767844.530072:0:9316:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942077cd600. 02000000:00000001:3.0:1775767844.530074:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767844.530079:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767844.530083:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00020000:00000010:2.0:1775767844.530085:0:9316:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff99420876d680. 02000000:00000001:3.0:1775767844.530086:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:2.0:1775767844.530090:0:9316:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076cf60. 00000080:00000010:2.0:1775767844.530093:0:9316:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff994205066870. 00000800:00000200:0.1:1775767844.530094:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 02000000:00000010:3.0:1775767844.530095:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2c00. 00000020:00000010:2.0:1775767844.530096:0:9316:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994209d2cc00. 00000800:00000200:0.1:1775767844.530100:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 52 secs later) 00000020:00000010:2.0:1775767844.530101:0:9316:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994210b32630. 00000100:00000001:3.0:1775767844.530102:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775767844.530106:0:9316:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:2.0:1775767844.530109:0:9316:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767844.530114:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a9180 x1862022294242176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000080:00000001:2.0:1775767844.530117:0:9316:0:(rw.c:1481:ll_writepages()) Process entered 00000080:00000001:2.0:1775767844.530120:0:9316:0:(file.c:5250:cl_sync_file_range()) Process entered 00000020:00000001:2.0:1775767844.530122:0:9316:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775767844.530127:0:9316:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994210b32630. 00000020:00000010:2.0:1775767844.530132:0:9316:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9942060d5200. 00000100:00000001:3.0:1775767844.530135:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000020:00000010:2.0:1775767844.530136:0:9316:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff9942060d4c00. 00000100:00000001:3.0:1775767844.530138:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1775767844.530140:0:9316:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942182d6240. 00000100:00000040:3.0:1775767844.530145:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bdc2586] to pc [ptlrpcd_rcv+-1] req@ffff9942099a9180 x1862022294242176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000010:2.0:1775767844.530146:0:9316:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207ccc2f0. 00020000:00000010:2.0:1775767844.530151:0:9316:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994220156e40. 00008000:00000010:2.0:1775767844.530187:0:9316:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942077a8e20. 00000080:00000010:2.0:1775767844.530193:0:9316:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff9942087472a0. 00000080:00000010:2.0:1775767844.530198:0:9316:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b8000. 00000100:00000001:3.0:1775767844.530200:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1775767844.530203:0:9316:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9942060d4800. 00000100:00080000:3.0:1775767844.530205:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000008:00000010:2.0:1775767844.530210:0:9316:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942077cd600. 00000100:00080000:3.0:1775767844.530213:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00020000:00000010:2.0:1775767844.530214:0:9316:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff99420876d680. 00000100:00000001:3.0:1775767844.530217:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00008000:00000010:2.0:1775767844.530220:0:9316:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076cf60. 00000100:00080000:3.0:1775767844.530220:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000080:00000010:2.0:1775767844.530223:0:9316:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff994205066870. 00000020:00000001:2.0:1775767844.530231:0:9316:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107759908408 : -112965949643208 : ffff994210b32638) 00000100:00000001:3.0:1775767844.530232:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000020:00001000:2.0:1775767844.530235:0:9316:0:(cl_object.c:831:cl_env_get()) 1@ffff994210b32630 00000100:00080000:3.0:1775767844.530238:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5096 00000020:00000001:2.0:1775767844.530239:0:9316:0:(cl_object.c:151:cl_object_top()) ffff9942078a7f58 -> ffff9942078a7f58 00000020:00000001:2.0:1775767844.530243:0:9316:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:2.0:1775767844.530246:0:9316:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000400:00000200:3.0:1775767844.530248:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000080:00200000:2.0:1775767844.530249:0:9316:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017b:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000400:00000200:3.0:1775767844.530255:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000020:00000001:2.0:1775767844.530256:0:9316:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775767844.530259:0:9316:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00080000:3.0:1775767844.530260:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000080:00000001:2.0:1775767844.530262:0:9316:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767844.530267:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00020000:00000002:2.0:1775767844.530268:0:9316:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017b:0x0]io ffff9942077b8060 type 6 ignore/verify layout 1/0 00000100:00000001:3.0:1775767844.530270:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00020000:00000002:2.0:1775767844.530273:0:9316:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207c1f000) owner 0000000000000000/ffff994201384f80 00000100:00000040:3.0:1775767844.530276:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00020000:00000001:2.0:1775767844.530277:0:9316:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:2.0:1775767844.530280:0:9316:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00000100:00000001:3.0:1775767844.530281:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00020000:00000001:2.0:1775767844.530282:0:9316:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00020000:00000001:2.0:1775767844.530286:0:9316:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767844.530286:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00020000:00000001:2.0:1775767844.530288:0:9316:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:2.0:1775767844.530297:0:9316:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00000020:00000040:3.0:1775767844.530297:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00020000:00000001:2.0:1775767844.530300:0:9316:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767844.530302:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000001:2.0:1775767844.530303:0:9316:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000040:3.0:1775767844.530305:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:2.0:1775767844.530306:0:9316:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:2.0:1775767844.530308:0:9316:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00020000:00000002:2.0:1775767844.530311:0:9316:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207c1f000) owner 0000000000000000/ffff994201384f80 00000020:00000001:3.0:1775767844.530312:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:2.0:1775767844.530313:0:9316:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767844.530319:0:9316:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:3.0:1775767844.530319:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:2.0:1775767844.530322:0:9316:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00020000:00000001:2.0:1775767844.530326:0:9316:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00000100:00000040:3.0:1775767844.530326:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00020000:00200000:2.0:1775767844.530330:0:9316:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00000001:3.0:1775767844.530331:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00020000:00000002:2.0:1775767844.530335:0:9316:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00000100:00000040:3.0:1775767844.530337:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00020000:00000001:2.0:1775767844.530342:0:9316:0:(lov_io.c:155:lov_sub_get()) Process entered 00000100:00000001:3.0:1775767844.530343:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00020000:00000001:2.0:1775767844.530346:0:9316:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000020:00000040:3.0:1775767844.530348:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000020:00000001:2.0:1775767844.530349:0:9316:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775767844.530353:0:9316:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994210b32090. 00000020:00000010:2.0:1775767844.530361:0:9316:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9942060d5800. 00000100:00080000:3.0:1775767844.530365:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000020:00000010:2.0:1775767844.530366:0:9316:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff9942060d5a00. 00000100:00000001:3.0:1775767844.530368:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1775767844.530373:0:9316:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942182d64e0. 02000000:00000001:3.0:1775767844.530373:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767844.530378:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00000010:2.0:1775767844.530380:0:9316:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207cc8b28. 02000000:00000001:3.0:1775767844.530382:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:2.0:1775767844.530385:0:9316:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994220156840. 00000100:00000001:3.0:1775767844.530386:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767844.530390:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00008000:00000010:2.0:1775767844.530391:0:9316:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942077a9530. 00000080:00000010:2.0:1775767844.530408:0:9316:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994208747738. 00000008:00000001:3.0:1775767844.530409:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767844.530414:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:2.0:1775767844.530416:0:9316:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077ba910. 00000100:00000010:3.0:1775767844.530419:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8700. 00000020:00000010:2.0:1775767844.530423:0:9316:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994209d2cc00. 00000020:00000040:3.0:1775767844.530424:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00000008:00000010:2.0:1775767844.530429:0:9316:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942077cd8c0. 02000000:00000001:3.0:1775767844.530430:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767844.530433:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767844.530435:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00020000:00000010:2.0:1775767844.530436:0:9316:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff99420876e400. 02000000:00000001:3.0:1775767844.530438:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00008000:00000010:2.0:1775767844.530441:0:9316:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076c3f0. 02000000:00000001:3.0:1775767844.530441:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:2.0:1775767844.530448:0:9316:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942050663c0. 02000000:00000010:3.0:1775767844.530450:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0400. 00000020:00000001:2.0:1775767844.530455:0:9316:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107759906968 : -112965949644648 : ffff994210b32098) 00000100:00000001:3.0:1775767844.530456:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:2.0:1775767844.530462:0:9316:0:(cl_object.c:831:cl_env_get()) 1@ffff994210b32090 00000100:00100000:3.0:1775767844.530466:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8700 x1862022294242304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000001:2.0:1775767844.530468:0:9316:0:(cl_object.c:151:cl_object_top()) ffff9942078a75f8 -> ffff9942078a7f58 00000020:00000001:2.0:1775767844.530474:0:9316:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:2.0:1775767844.530478:0:9316:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775767844.530482:0:9316:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:2.0:1775767844.530486:0:9316:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767844.530486:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767844.530491:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767844.530492:0:9316:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767844.530498:0:9316:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107621738216 : -112966087813400 : ffff99420876d6e8) 00000100:00000040:3.0:1775767844.530498:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000faa25e3b] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8700 x1862022294242304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1775767844.530504:0:9316:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000020:00000001:2.0:1775767844.530508:0:9316:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767844.530513:0:9316:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767844.530519:0:9316:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767844.530524:0:9316:0:(cl_io.c:314:cl_io_lock()) Process entered 00020000:00000001:2.0:1775767844.530528:0:9316:0:(lov_io.c:1108:lov_io_lock()) Process entered 00000100:00000001:3.0:1775767844.530530:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767844.530531:0:9316:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775767844.530534:0:9316:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:2.0:1775767844.530537:0:9316:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000100:00080000:3.0:1775767844.530538:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000001:2.0:1775767844.530540:0:9316:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767844.530544:0:9316:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767844.530548:0:9316:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767844.530550:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00020000:00000001:2.0:1775767844.530553:0:9316:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767844.530555:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000020:00000001:2.0:1775767844.530556:0:9316:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:2.0:1775767844.530560:0:9316:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767844.530560:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000020:00000001:2.0:1775767844.530564:0:9316:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767844.530567:0:9316:0:(cl_io.c:528:cl_io_start()) Process entered 00000100:00000001:3.0:1775767844.530567:0:13471:0:(import.c:484:import_select_connection()) Process entered 00020000:00000001:2.0:1775767844.530570:0:9316:0:(lov_io.c:1114:lov_io_start()) Process entered 00020000:00000001:2.0:1775767844.530572:0:9316:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00080000:3.0:1775767844.530574:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5096 00000020:00000001:2.0:1775767844.530575:0:9316:0:(cl_io.c:528:cl_io_start()) Process entered 00000008:00000001:2.0:1775767844.530578:0:9316:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000008:00000001:2.0:1775767844.530581:0:9316:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000008:00000001:2.0:1775767844.530586:0:9316:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000400:00000200:3.0:1775767844.530586:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00000020:2.0:1775767844.530589:0:9316:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:2.0:1775767844.530592:0:9316:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1775767844.530595:0:9316:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000400:00000200:3.0:1775767844.530596:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000008:00000001:2.0:1775767844.530597:0:9316:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000008:00000001:2.0:1775767844.530600:0:9316:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107884688256 : -112965824863360 : ffff994218232380) 00000100:00080000:3.0:1775767844.530602:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000008:00000020:2.0:1775767844.530603:0:9316:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994218232380 ready 0|-|- wr 1|-|+ rd 0|- 9 in flight 00000008:00000001:2.0:1775767844.530611:0:9316:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000100:00000040:3.0:1775767844.530612:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000008:00000020:2.0:1775767844.530613:0:9316:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:2.0:1775767844.530616:0:9316:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767844.530619:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000008:00000001:2.0:1775767844.530620:0:9316:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000008:00000020:2.0:1775767844.530624:0:9316:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994210674fc0 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000100:00000040:3.0:1775767844.530627:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000008:00000001:2.0:1775767844.530631:0:9316:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767844.530633:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000008:00000001:2.0:1775767844.530635:0:9316:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767844.530654:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000001:2.0:1775767844.530657:0:9316:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775767844.530659:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00020000:00000001:2.0:1775767844.530660:0:9316:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767844.530663:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00020000:00000001:2.0:1775767844.530670:0:9316:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775767844.530670:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:2.0:1775767844.530676:0:9316:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767844.530677:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:2.0:1775767844.530681:0:9316:0:(cl_io.c:555:cl_io_end()) Process entered 00020000:00000001:2.0:1775767844.530685:0:9316:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00000020:00000001:3.0:1775767844.530685:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00020000:00000001:2.0:1775767844.530688:0:9316:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000020:00000001:2.0:1775767844.530691:0:9316:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000001:2.0:1775767844.530694:0:9316:0:(cl_io.c:563:cl_io_end()) Process leaving 00020000:00000001:2.0:1775767844.530697:0:9316:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767844.530697:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00020000:00000001:2.0:1775767844.530704:0:9316:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000100:00000001:3.0:1775767844.530704:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:2.0:1775767844.530708:0:9316:0:(cl_io.c:563:cl_io_end()) Process leaving 00000020:00000001:2.0:1775767844.530712:0:9316:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000100:00000040:3.0:1775767844.530712:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00020000:00000001:2.0:1775767844.530715:0:9316:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00020000:00000001:2.0:1775767844.530719:0:9316:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000001:3.0:1775767844.530720:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775767844.530723:0:9316:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000020:00000040:3.0:1775767844.530726:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000020:00000001:2.0:1775767844.530727:0:9316:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:2.0:1775767844.530731:0:9316:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767844.530733:0:9316:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767844.530734:0:9316:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000020:00000001:2.0:1775767844.530735:0:9316:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:2.0:1775767844.530740:0:9316:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000100:00080000:3.0:1775767844.530741:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000020:00000001:2.0:1775767844.530744:0:9316:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000001:2.0:1775767844.530748:0:9316:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:2.0:1775767844.530750:0:9316:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00000100:00000001:3.0:1775767844.530750:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767844.530754:0:9316:0:(lov_io.c:1086:lov_io_call()) Process entered 02000000:00000001:3.0:1775767844.530757:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000020:00000001:2.0:1775767844.530758:0:9316:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000001:2.0:1775767844.530763:0:9316:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 02000000:00000001:3.0:1775767844.530766:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:2.0:1775767844.530767:0:9316:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767844.530775:0:9316:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767844.530775:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767844.530781:0:9316:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000100:00000001:3.0:1775767844.530783:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000020:00000001:2.0:1775767844.530784:0:9316:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000020:00000001:2.0:1775767844.530788:0:9316:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1775767844.530789:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000020:00000001:2.0:1775767844.530790:0:9316:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:2.0:1775767844.530793:0:9316:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:2.0:1775767844.530797:0:9316:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000008:00000001:3.0:1775767844.530798:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767844.530800:0:9316:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:2.0:1775767844.530803:0:9316:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000100:00000001:3.0:1775767844.530805:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:2.0:1775767844.530807:0:9316:0:(cl_object.c:925:cl_env_put()) 1@ffff994210b32090 00000020:00000010:2.0:1775767844.530813:0:9316:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff9942060d5a00. 00000100:00000010:3.0:1775767844.530814:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8380. 00000020:00000040:3.0:1775767844.530835:0:13471:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000010:2.0:1775767844.530837:0:9316:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942182d64e0. 02000000:00000001:3.0:1775767844.530839:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767844.530842:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:2.0:1775767844.530843:0:9316:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207cc8b28. 00000100:00000001:3.0:1775767844.530847:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00020000:00000010:2.0:1775767844.530849:0:9316:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994220156840. 02000000:00000001:3.0:1775767844.530849:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767844.530853:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:2.0:1775767844.530855:0:9316:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942077a9530. 02000000:00000010:3.0:1775767844.530859:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2000. 00000080:00000010:2.0:1775767844.530860:0:9316:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994208747738. 00000080:00000010:2.0:1775767844.530865:0:9316:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077ba910. 00000100:00000001:3.0:1775767844.530866:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1775767844.530871:0:9316:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9942060d5800. 00000008:00000010:2.0:1775767844.530875:0:9316:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942077cd8c0. 00000100:00100000:3.0:1775767844.530875:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8380 x1862022294242432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00020000:00000010:2.0:1775767844.530882:0:9316:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff99420876e400. 00008000:00000010:2.0:1775767844.530887:0:9316:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076c3f0. 00000100:00000001:3.0:1775767844.530892:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000080:00000010:2.0:1775767844.530893:0:9316:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942050663c0. 00000100:00000001:3.0:1775767844.530896:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1775767844.530897:0:9316:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994209d2cc00. 00000020:00000010:2.0:1775767844.530902:0:9316:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994210b32090. 00000100:00000040:3.0:1775767844.530903:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055f5621a] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8380 x1862022294242432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:2.0:1775767844.530905:0:9316:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:2.0:1775767844.530908:0:9316:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:2.0:1775767844.530911:0:9316:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000080:00200000:2.0:1775767844.530914:0:9316:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:2.0:1775767844.530921:0:9316:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:2.0:1775767844.530924:0:9316:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:2.0:1775767844.530928:0:9316:0:(cl_object.c:925:cl_env_put()) 1@ffff994210b32630 00000100:00000001:3.0:1775767844.530929:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1775767844.530933:0:9316:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff9942060d4c00. 00000100:00000040:3.0:1775767844.530936:0:13471:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5107) 00000020:00000010:2.0:1775767844.530939:0:9316:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942182d6240. 00000008:00000010:2.0:1775767844.530942:0:9316:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207ccc2f0. 00020000:00000010:2.0:1775767844.530945:0:9316:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994220156e40. 00008000:00000010:2.0:1775767844.530949:0:9316:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942077a8e20. 00000080:00000010:2.0:1775767844.530955:0:9316:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff9942087472a0. 00000080:00000010:2.0:1775767844.530959:0:9316:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b8000. 00000020:00000010:2.0:1775767844.530963:0:9316:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9942060d5200. 00000008:00000010:2.0:1775767844.530966:0:9316:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942077cd600. 00020000:00000010:2.0:1775767844.530968:0:9316:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff99420876d680. 00008000:00000010:2.0:1775767844.530972:0:9316:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076cf60. 00000080:00000010:2.0:1775767844.530975:0:9316:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff994205066870. 00000020:00000010:2.0:1775767844.530979:0:9316:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9942060d4800. 00000020:00000010:2.0:1775767844.530982:0:9316:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994210b32630. 00000080:00000001:2.0:1775767844.530985:0:9316:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:2.0:1775767844.530988:0:9316:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775767844.530995:0:9316:0:(rw.c:1481:ll_writepages()) Process entered 00000080:00000001:2.0:1775767844.530997:0:9316:0:(file.c:5250:cl_sync_file_range()) Process entered 00000020:00000001:2.0:1775767844.530999:0:9316:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775767844.531002:0:9316:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994210b32630. 00000020:00000010:2.0:1775767844.531005:0:9316:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994209d2cc00. 00000020:00000010:2.0:1775767844.531009:0:9316:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994209d2dc00. 00000020:00000010:2.0:1775767844.531012:0:9316:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942182d6240. 00000008:00000010:2.0:1775767844.531014:0:9316:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207ccc2f0. 00020000:00000010:2.0:1775767844.531018:0:9316:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994220156e40. 00008000:00000010:2.0:1775767844.531020:0:9316:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942077a8e20. 00000080:00000010:2.0:1775767844.531024:0:9316:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff9942087472a0. 00000080:00000010:2.0:1775767844.531027:0:9316:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b8000. 00000020:00000010:2.0:1775767844.531030:0:9316:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994209d2d600. 00000008:00000010:2.0:1775767844.531034:0:9316:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942077cd600. 00020000:00000010:2.0:1775767844.531036:0:9316:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff99420876d680. 00008000:00000010:2.0:1775767844.531039:0:9316:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076cf60. 00000080:00000010:2.0:1775767844.531045:0:9316:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff994205066870. 00000020:00000001:2.0:1775767844.531050:0:9316:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107759908408 : -112965949643208 : ffff994210b32638) 00000020:00001000:2.0:1775767844.531055:0:9316:0:(cl_object.c:831:cl_env_get()) 1@ffff994210b32630 00000020:00000001:2.0:1775767844.531058:0:9316:0:(cl_object.c:151:cl_object_top()) ffff99420791a850 -> ffff99420791a850 00000020:00000001:2.0:1775767844.531060:0:9316:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:2.0:1775767844.531062:0:9316:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:2.0:1775767844.531065:0:9316:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017f:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000800:00000200:0.1:1775767844.531069:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000001:2.0:1775767844.531070:0:9316:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775767844.531072:0:9316:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000800:00000200:0.1:1775767844.531073:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 52 secs later) 00000080:00000001:2.0:1775767844.531074:0:9316:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775767844.531077:0:9316:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017f:0x0]io ffff9942077b8060 type 6 ignore/verify layout 1/0 00020000:00000002:2.0:1775767844.531079:0:9316:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207dfbd90) owner 0000000000000000/ffff994201384f80 00020000:00000001:2.0:1775767844.531083:0:9316:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:2.0:1775767844.531085:0:9316:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:2.0:1775767844.531087:0:9316:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00020000:00000001:2.0:1775767844.531089:0:9316:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767844.531091:0:9316:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:2.0:1775767844.531096:0:9316:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:2.0:1775767844.531098:0:9316:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767844.531101:0:9316:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775767844.531103:0:9316:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:2.0:1775767844.531105:0:9316:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00020000:00000002:2.0:1775767844.531107:0:9316:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207dfbd90) owner 0000000000000000/ffff994201384f80 00000020:00000001:2.0:1775767844.531108:0:9316:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767844.531112:0:9316:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:2.0:1775767844.531113:0:9316:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00020000:00000001:2.0:1775767844.531115:0:9316:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00020000:00200000:2.0:1775767844.531117:0:9316:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00000002:2.0:1775767844.531119:0:9316:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:2.0:1775767844.531122:0:9316:0:(lov_io.c:155:lov_sub_get()) Process entered 00020000:00000001:2.0:1775767844.531123:0:9316:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000020:00000001:2.0:1775767844.531125:0:9316:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775767844.531127:0:9316:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994210b32090. 00000020:00000010:2.0:1775767844.531130:0:9316:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994209d2de00. 00000020:00000010:2.0:1775767844.531134:0:9316:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994209d2d000. 00000020:00000010:2.0:1775767844.531138:0:9316:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942182d64e0. 00000008:00000010:2.0:1775767844.531142:0:9316:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207cc8b28. 00020000:00000010:2.0:1775767844.531145:0:9316:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994220156840. 00008000:00000010:2.0:1775767844.531147:0:9316:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942077a9530. 00000080:00000010:2.0:1775767844.531151:0:9316:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994208747738. 00000080:00000010:2.0:1775767844.531155:0:9316:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077ba910. 00000020:00000010:2.0:1775767844.531159:0:9316:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9942060d4800. 00000008:00000010:2.0:1775767844.531163:0:9316:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942077cd8c0. 00020000:00000010:2.0:1775767844.531167:0:9316:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff99420876e400. 00008000:00000010:2.0:1775767844.531172:0:9316:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076c3f0. 00000080:00000010:2.0:1775767844.531175:0:9316:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942050663c0. 00000020:00000001:2.0:1775767844.531178:0:9316:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107759906968 : -112965949644648 : ffff994210b32098) 00000020:00001000:2.0:1775767844.531180:0:9316:0:(cl_object.c:831:cl_env_get()) 1@ffff994210b32090 00000020:00000001:2.0:1775767844.531182:0:9316:0:(cl_object.c:151:cl_object_top()) ffff99420791a210 -> ffff99420791a850 00000020:00000001:2.0:1775767844.531184:0:9316:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:2.0:1775767844.531205:0:9316:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775767844.531207:0:9316:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:2.0:1775767844.531209:0:9316:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767844.531211:0:9316:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767844.531216:0:9316:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107621738216 : -112966087813400 : ffff99420876d6e8) 00000020:00000001:2.0:1775767844.531219:0:9316:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000020:00000001:2.0:1775767844.531221:0:9316:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767844.531225:0:9316:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767844.531229:0:9316:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767844.531231:0:9316:0:(cl_io.c:314:cl_io_lock()) Process entered 00020000:00000001:2.0:1775767844.531233:0:9316:0:(lov_io.c:1108:lov_io_lock()) Process entered 00020000:00000001:2.0:1775767844.531236:0:9316:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775767844.531239:0:9316:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:2.0:1775767844.531240:0:9316:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:2.0:1775767844.531243:0:9316:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767844.531247:0:9316:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767844.531251:0:9316:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767844.531254:0:9316:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767844.531257:0:9316:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:2.0:1775767844.531260:0:9316:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767844.531261:0:9316:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767844.531264:0:9316:0:(cl_io.c:528:cl_io_start()) Process entered 00020000:00000001:2.0:1775767844.531266:0:9316:0:(lov_io.c:1114:lov_io_start()) Process entered 00020000:00000001:2.0:1775767844.531269:0:9316:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775767844.531270:0:9316:0:(cl_io.c:528:cl_io_start()) Process entered 00000008:00000001:2.0:1775767844.531273:0:9316:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000008:00000001:2.0:1775767844.531274:0:9316:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000008:00000001:2.0:1775767844.531276:0:9316:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:2.0:1775767844.531279:0:9316:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:2.0:1775767844.531281:0:9316:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1775767844.531284:0:9316:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000008:00000001:2.0:1775767844.531286:0:9316:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000008:00000001:2.0:1775767844.531287:0:9316:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107754938304 : -112965954613312 : ffff994210674fc0) 00000008:00000020:2.0:1775767844.531291:0:9316:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994210674fc0 ready 0|-|- wr 1|-|+ rd 0|- 9 in flight 00000008:00000001:2.0:1775767844.531296:0:9316:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:2.0:1775767844.531298:0:9316:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:2.0:1775767844.531300:0:9316:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1775767844.531304:0:9316:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000008:00000020:2.0:1775767844.531306:0:9316:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994218232380 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000008:00000001:2.0:1775767844.531311:0:9316:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1775767844.531313:0:9316:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767844.531317:0:9316:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767844.531322:0:9316:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767844.531327:0:9316:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767844.531330:0:9316:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767844.531333:0:9316:0:(cl_io.c:555:cl_io_end()) Process entered 00020000:00000001:2.0:1775767844.531336:0:9316:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00020000:00000001:2.0:1775767844.531339:0:9316:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000020:00000001:2.0:1775767844.531341:0:9316:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000001:2.0:1775767844.531343:0:9316:0:(cl_io.c:563:cl_io_end()) Process leaving 00020000:00000001:2.0:1775767844.531347:0:9316:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767844.531351:0:9316:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000020:00000001:2.0:1775767844.531353:0:9316:0:(cl_io.c:563:cl_io_end()) Process leaving 00000020:00000001:2.0:1775767844.531356:0:9316:0:(cl_io.c:352:cl_io_unlock()) Process entered 00020000:00000001:2.0:1775767844.531359:0:9316:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00020000:00000001:2.0:1775767844.531362:0:9316:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775767844.531364:0:9316:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000020:00000001:2.0:1775767844.531367:0:9316:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:2.0:1775767844.531369:0:9316:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767844.531375:0:9316:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767844.531380:0:9316:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000020:00000001:2.0:1775767844.531382:0:9316:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:2.0:1775767844.531384:0:9316:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000020:00000001:2.0:1775767844.531386:0:9316:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000001:2.0:1775767844.531388:0:9316:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:2.0:1775767844.531391:0:9316:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00020000:00000001:2.0:1775767844.531393:0:9316:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775767844.531395:0:9316:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000001:2.0:1775767844.531396:0:9316:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00020000:00000001:2.0:1775767844.531412:0:9316:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767844.531417:0:9316:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767844.531420:0:9316:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000020:00000001:2.0:1775767844.531423:0:9316:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000020:00000001:2.0:1775767844.531426:0:9316:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767844.531430:0:9316:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:2.0:1775767844.531433:0:9316:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:2.0:1775767844.531436:0:9316:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:2.0:1775767844.531439:0:9316:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:2.0:1775767844.531441:0:9316:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:2.0:1775767844.531444:0:9316:0:(cl_object.c:925:cl_env_put()) 1@ffff994210b32090 00000020:00000010:2.0:1775767844.531448:0:9316:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994209d2d000. 00000020:00000010:2.0:1775767844.531453:0:9316:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942182d64e0. 00000008:00000010:2.0:1775767844.531458:0:9316:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207cc8b28. 00020000:00000010:2.0:1775767844.531461:0:9316:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994220156840. 00008000:00000010:2.0:1775767844.531465:0:9316:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942077a9530. 00000080:00000010:2.0:1775767844.531470:0:9316:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994208747738. 00000080:00000010:2.0:1775767844.531475:0:9316:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077ba910. 00000020:00000010:2.0:1775767844.531480:0:9316:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994209d2de00. 00000008:00000010:2.0:1775767844.531486:0:9316:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942077cd8c0. 00020000:00000010:2.0:1775767844.531492:0:9316:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff99420876e400. 00008000:00000010:2.0:1775767844.531498:0:9316:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076c3f0. 00000080:00000010:2.0:1775767844.531504:0:9316:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942050663c0. 00000020:00000010:2.0:1775767844.531509:0:9316:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9942060d4800. 00000020:00000010:2.0:1775767844.531514:0:9316:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994210b32090. 00020000:00000001:2.0:1775767844.531519:0:9316:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:2.0:1775767844.531524:0:9316:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:2.0:1775767844.531527:0:9316:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000080:00200000:2.0:1775767844.531529:0:9316:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017f:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:2.0:1775767844.531534:0:9316:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:2.0:1775767844.531536:0:9316:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:2.0:1775767844.531540:0:9316:0:(cl_object.c:925:cl_env_put()) 1@ffff994210b32630 00000020:00000010:2.0:1775767844.531544:0:9316:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994209d2dc00. 00000020:00000010:2.0:1775767844.531551:0:9316:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942182d6240. 00000008:00000010:2.0:1775767844.531557:0:9316:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207ccc2f0. 00020000:00000010:2.0:1775767844.531565:0:9316:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994220156e40. 00008000:00000010:2.0:1775767844.531571:0:9316:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942077a8e20. 00000080:00000010:2.0:1775767844.531577:0:9316:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff9942087472a0. 00000080:00000010:2.0:1775767844.531583:0:9316:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b8000. 00000020:00000010:2.0:1775767844.531590:0:9316:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994209d2cc00. 00000008:00000010:2.0:1775767844.531596:0:9316:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942077cd600. 00020000:00000010:2.0:1775767844.531603:0:9316:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff99420876d680. 00008000:00000010:2.0:1775767844.531609:0:9316:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076cf60. 00000080:00000010:2.0:1775767844.531614:0:9316:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff994205066870. 00000020:00000010:2.0:1775767844.531620:0:9316:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994209d2d600. 00000020:00000010:2.0:1775767844.531626:0:9316:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994210b32630. 00000080:00000001:2.0:1775767844.531632:0:9316:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:2.0:1775767844.531636:0:9316:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775767844.531637:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767844.531641:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1775767844.531823:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767844.531824:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1775767844.532088:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767844.532090:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 52 secs later) 00000020:00000001:2.0:1775767844.593303:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767844.593308:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767844.593312:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775767845.553224:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767845.553236:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767845.553238:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767845.553239:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767845.553244:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:1.0:1775767845.553247:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767845.553247:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767845.553253:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1775767845.553254:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767845.553256:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767845.553259:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767845.553261:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767845.553263:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767845.553269:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767845.553272:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767845.553275:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767845.553277:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767845.553280:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767845.873228:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299770686, at 4299770688 00000100:00000010:1.0:1775767845.873240:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3000. 00000100:00000001:1.0:1775767846.577232:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767846.577232:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767846.577239:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767846.577240:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767846.577242:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767846.577244:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767846.577247:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775767846.577249:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767846.577251:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767846.577254:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767846.577255:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767846.577258:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767846.577258:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767846.577262:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767846.577265:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767846.577277:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767846.577279:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767846.577281:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767846.641316:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767846.641319:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767846.641322:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775767847.601268:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767847.601274:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767847.601277:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767847.601281:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767847.601281:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767847.601284:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767847.601290:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767847.601295:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1775767847.601298:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775767847.601299:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767847.601301:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767847.601303:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767847.601304:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767847.601335:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767847.601337:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767847.601337:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767847.601340:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767847.601343:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767847.601345:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767847.601351:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767847.601352:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767847.601353:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767847.601357:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767847.601362:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767847.601363:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767847.601388:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767847.601408:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767848.625218:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767848.625224:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767848.625225:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767848.625227:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:3.0:1775767848.625230:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767848.625232:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767848.625234:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767848.625236:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767848.625238:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767848.625243:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00100000:1.0:1775767848.625244:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767848.625245:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767848.625250:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767848.625250:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767848.625255:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767848.625259:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767848.625261:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767848.625265:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767848.689375:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767848.689379:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767848.689383:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775767849.649230:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767849.649236:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767849.649238:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767849.649243:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767849.649246:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767849.649249:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767849.649251:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767849.649253:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767849.649257:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767849.649257:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767849.649263:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767849.649266:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767849.649269:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775767849.649273:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767849.649276:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775767849.649277:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775767849.649282:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767849.649285:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775767849.649285:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775767849.649288:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767849.649290:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767849.649294:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767849.649298:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767849.649305:0:13471:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5102 00000400:00000200:3.0:1775767849.649315:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767849.649321:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767849.649324:0:13471:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767849.649328:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767849.649331:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767849.649334:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767849.649336:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767849.649340:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767849.649343:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:3.0:1775767849.649345:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767849.649395:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:3.0:1775767849.649397:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:3.0:1775767849.649399:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:3.0:1775767849.649402:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767849.649403:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767849.649405:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767849.649407:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767849.649409:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:3.0:1775767849.649411:0:13471:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767849.649413:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767849.649415:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767849.649418:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767849.649420:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767849.649421:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775767849.649423:0:13471:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767849.649431:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8380. 00000020:00000040:3.0:1775767849.649434:0:13471:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:3.0:1775767849.649436:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767849.649438:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767849.649439:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767849.649442:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767849.649445:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767849.649453:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0000. 00000100:00000001:3.0:1775767849.649459:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767849.649467:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8380 x1862022294242560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767849.649474:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767849.649478:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767849.649481:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055f5621a] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8380 x1862022294242560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767849.649498:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767849.649501:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767849.649505:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775767849.649507:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767849.649508:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767849.649511:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767849.649514:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5102 00000400:00000200:3.0:1775767849.649519:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767849.649522:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767849.649525:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767849.649527:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767849.649530:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767849.649531:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767849.649534:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767849.649536:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767849.649537:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:3.0:1775767849.649539:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767849.649541:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:3.0:1775767849.649543:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:3.0:1775767849.649546:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:3.0:1775767849.649549:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767849.649551:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767849.649553:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767849.649554:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767849.649557:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00080000:3.0:1775767849.649559:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767849.649562:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767849.649564:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767849.649569:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767849.649571:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767849.649573:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767849.649577:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775767849.649580:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767849.649582:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767849.649585:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8700. 00000020:00000040:3.0:1775767849.649587:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:3.0:1775767849.649590:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767849.649591:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767849.649593:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767849.649595:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767849.649596:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767849.649599:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2800. 00000100:00000001:3.0:1775767849.649602:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767849.649607:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8700 x1862022294242688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767849.649614:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767849.649616:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767849.649621:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000faa25e3b] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8700 x1862022294242688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767849.649629:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767849.649632:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767849.649635:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775767849.649637:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767849.649638:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767849.649641:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767849.649643:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5102 00000400:00000200:3.0:1775767849.649646:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767849.649649:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767849.649651:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767849.649653:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767849.649655:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767849.649657:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767849.649659:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767849.649662:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767849.649663:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:3.0:1775767849.649665:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767849.649666:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:3.0:1775767849.649669:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:3.0:1775767849.649671:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:3.0:1775767849.649674:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767849.649676:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767849.649678:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767849.649680:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767849.649682:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:3.0:1775767849.649684:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767849.649686:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767849.649688:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767849.649691:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767849.649693:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767849.649695:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767849.649697:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775767849.649699:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767849.649701:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767849.649704:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a9180. 00000020:00000040:3.0:1775767849.649706:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:3.0:1775767849.649709:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767849.649710:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767849.649712:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767849.649713:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767849.649714:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775767849.649716:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 02000000:00000010:3.0:1775767849.649718:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0c00. 00000800:00000200:0.1:1775767849.649720:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000100:00000001:3.0:1775767849.649721:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767849.649725:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a9180 x1862022294242816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767849.649733:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767849.649735:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767849.649738:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bdc2586] to pc [ptlrpcd_rcv+-1] req@ffff9942099a9180 x1862022294242816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767849.649746:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767849.649748:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767849.649752:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775767849.649753:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767849.649755:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767849.649758:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767849.649760:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5102 00000400:00000200:3.0:1775767849.649764:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767849.649766:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767849.649768:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767849.649771:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767849.649774:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767849.649776:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767849.649778:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767849.649779:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767849.649781:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:3.0:1775767849.649782:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767849.649784:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:3.0:1775767849.649786:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:3.0:1775767849.649788:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:3.0:1775767849.649791:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767849.649793:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767849.649795:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767849.649797:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767849.649798:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:3.0:1775767849.649801:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767849.649803:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767849.649805:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767849.649808:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767849.649810:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767849.649812:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767849.649814:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:3.0:1775767849.649817:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767849.649819:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767849.649821:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ab480. 00000020:00000040:3.0:1775767849.649824:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:3.0:1775767849.649826:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767849.649827:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767849.649829:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767849.649831:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767849.649832:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767849.649836:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2000. 00000100:00000001:3.0:1775767849.649838:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767849.649842:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ab480 x1862022294242944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767849.649850:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767849.649852:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767849.649855:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000067a0ba74] to pc [ptlrpcd_rcv+-1] req@ffff9942099ab480 x1862022294242944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767849.649865:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767849.649868:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767849.649871:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775767849.649873:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767849.649874:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767849.649877:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767849.649879:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5102 00000400:00000200:3.0:1775767849.649882:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767849.649885:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767849.649888:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767849.649890:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767849.649893:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767849.649895:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767849.649897:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767849.649900:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767849.649902:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:3.0:1775767849.649903:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767849.649914:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:3.0:1775767849.649916:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:3.0:1775767849.649918:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:3.0:1775767849.649921:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767849.649923:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767849.649925:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767849.649928:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767849.649930:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:3.0:1775767849.649933:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767849.649935:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767849.649936:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767849.649939:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767849.649941:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767849.649943:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767849.649945:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:3.0:1775767849.649948:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767849.649950:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767849.649953:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8e00. 00000020:00000040:3.0:1775767849.649955:0:13471:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:3.0:1775767849.649957:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767849.649959:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767849.649960:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767849.649962:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767849.649964:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767849.649967:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0400. 00000100:00000001:3.0:1775767849.649969:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767849.649973:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8e00 x1862022294243072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767849.649986:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767849.649988:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767849.649992:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000864a6866] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8e00 x1862022294243072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767849.650001:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767849.650005:0:13471:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5112) 00000800:00000200:0.1:1775767849.650105:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767849.650107:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1775767849.650311:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767849.650313:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1775767849.650509:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767849.650511:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1775767849.650775:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767849.650776:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000100:00000001:1.0:1775767850.673268:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767850.673268:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767850.673276:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767850.673276:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767850.673279:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767850.673280:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767850.673284:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775767850.673290:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1775767850.673296:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767850.673301:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767850.673301:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767850.673306:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767850.673307:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767850.673309:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767850.673313:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767850.673313:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767850.673317:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767850.673321:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767850.737355:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767850.737359:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767850.737362:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775767850.993261:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299775805, at 4299775808 00000100:00000010:1.0:1775767850.993281:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3f00. 00000100:00000001:2.0:1775767851.697239:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767851.697242:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767851.697247:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767851.697249:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767851.697250:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767851.697252:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767851.697266:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:2.0:1775767851.697266:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775767851.697281:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767851.697282:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767851.697286:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767851.697287:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767851.697297:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767851.697300:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767851.697303:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767851.697304:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767851.697307:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767851.697316:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767852.721256:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767852.721256:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767852.721263:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767852.721263:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767852.721266:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767852.721267:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767852.721274:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:2.0:1775767852.721274:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775767852.721280:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767852.721281:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767852.721285:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767852.721286:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767852.721290:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767852.721293:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767852.721295:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767852.721296:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767852.721300:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767852.721303:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767852.785360:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767852.785363:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767852.785366:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775767853.745248:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767853.745249:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767853.745257:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767853.745257:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767853.745259:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767853.745260:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767853.745267:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:2.0:1775767853.745268:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767853.745276:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767853.745277:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767853.745282:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767853.745282:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767853.745289:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767853.745293:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767853.745296:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767853.745300:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767853.745304:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767853.745329:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767854.769270:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767854.769270:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767854.769276:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767854.769277:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767854.769278:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767854.769281:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767854.769281:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775767854.769283:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:1.0:1775767854.769285:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00080000:3.0:1775767854.769290:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775767854.769291:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767854.769292:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767854.769293:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767854.769297:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775767854.769312:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767854.769316:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775767854.769317:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767854.769322:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767854.769324:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775767854.769326:0:13471:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5107 00000100:00000001:1.0:1775767854.769327:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767854.769327:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767854.769330:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767854.769335:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767854.769336:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767854.769344:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767854.769348:0:13471:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767854.769353:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767854.769357:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767854.769361:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767854.769366:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767854.769370:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767854.769373:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:3.0:1775767854.769376:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767854.769379:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:3.0:1775767854.769383:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:3.0:1775767854.769386:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:3.0:1775767854.769391:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767854.769393:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775767854.769396:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000040:3.0:1775767854.769396:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767854.769399:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767854.769402:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00000001:2.0:1775767854.769413:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:3.0:1775767854.769414:0:13471:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775767854.769415:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767854.769418:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767854.769421:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767854.769421:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767854.769425:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1775767854.769425:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1775767854.769427:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767854.769429:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1775767854.769429:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767854.769434:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767854.769435:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767854.769437:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775767854.769440:0:13471:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767854.769448:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8a80. 00000020:00000040:3.0:1775767854.769450:0:13471:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:3.0:1775767854.769454:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767854.769455:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767854.769457:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767854.769460:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767854.769464:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767854.769471:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2c00. 00000100:00000001:3.0:1775767854.769477:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767854.769485:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8a80 x1862022294243200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767854.769493:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767854.769497:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767854.769500:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006c3265af] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8a80 x1862022294243200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767854.769524:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767854.769528:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767854.769532:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775767854.769534:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767854.769537:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767854.769541:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767854.769545:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5107 00000100:00000001:2.0:1775767854.769550:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:3.0:1775767854.769551:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767854.769553:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767854.769555:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000400:00000200:3.0:1775767854.769558:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767854.769561:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775767854.769564:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a8a80 x1862022294243200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775767854.769568:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767854.769572:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767854.769576:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767854.769580:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767854.769581:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:2.0:1775767854.769582:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767854.769583:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767854.769586:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 02000000:00000001:2.0:1775767854.769587:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:3.0:1775767854.769588:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000001:2.0:1775767854.769591:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:3.0:1775767854.769591:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:3.0:1775767854.769595:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 02000000:00000001:2.0:1775767854.769596:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767854.769599:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00100000:2.0:1775767854.769602:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294243200:192.168.202.156@tcp:250:kworker.0 00000100:00000040:3.0:1775767854.769605:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767854.769609:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775767854.769611:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000040:3.0:1775767854.769611:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767854.769615:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 02000000:00000001:2.0:1775767854.769616:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:3.0:1775767854.769617:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 02000000:00000001:2.0:1775767854.769618:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767854.769621:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 02000000:00000001:2.0:1775767854.769622:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1775767854.769623:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767854.769625:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000010:2.0:1775767854.769627:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f800. 02000000:00000001:2.0:1775767854.769630:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767854.769630:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767854.769635:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767854.769637:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:2.0:1775767854.769640:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00100000:3.0:1775767854.769642:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000400:00000010:2.0:1775767854.769643:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39c60. 00000008:00000001:3.0:1775767854.769645:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767854.769648:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775767854.769651:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f220. 00000100:00000010:3.0:1775767854.769652:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aa300. 00000020:00000040:3.0:1775767854.769656:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:3.0:1775767854.769658:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000200:2.0:1775767854.769659:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294243200, portal 25 02000000:00000001:3.0:1775767854.769659:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767854.769660:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1775767854.769663:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1775767854.769663:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1775767854.769666:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640855168 : -112966068696448 : ffff9942099a8a80) 02000000:00000001:3.0:1775767854.769667:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767854.769674:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b3400. 00000100:00000040:2.0:1775767854.769676:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a8a80 x1862022294243200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767879 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767854.769678:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767854.769685:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:3.0:1775767854.769686:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aa300 x1862022294243328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:2.0:1775767854.769690:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767854.769695:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000001:3.0:1775767854.769700:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000200:2.0:1775767854.769701:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294243200, offset 0 00000100:00000001:3.0:1775767854.769702:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767854.769707:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f33bdf15] to pc [ptlrpcd_rcv+-1] req@ffff9942099aa300 x1862022294243328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1775767854.769710:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00000001:3.0:1775767854.769719:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767854.769721:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00080000:3.0:1775767854.769723:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:2.0:1775767854.769727:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00080000:3.0:1775767854.769731:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000200:2.0:1775767854.769734:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1775767854.769735:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:2.0:1775767854.769738:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000100:00080000:3.0:1775767854.769738:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767854.769743:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:2.0:1775767854.769744:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767854.769750:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00080000:3.0:1775767854.769751:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5107 00000400:00000200:2.0:1775767854.769755:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225073:225073:256:4294967295] 192.168.202.156@tcp LPNI seq info [225069:225069:8:4294967295] 00000400:00000200:2.0:1775767854.769759:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767854.769770:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000400:00000200:3.0:1775767854.769770:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000800:00000200:2.0:1775767854.769780:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767854.769780:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767854.769785:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000800:00000010:2.0:1775767854.769786:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3300. 00000100:00000040:3.0:1775767854.769791:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000800:00000200:2.0:1775767854.769796:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:3.0:1775767854.769796:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:2.0:1775767854.769801:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000100:00000040:3.0:1775767854.769801:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767854.769804:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767854.769807:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767854.769810:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000800:00000200:2.1:1775767854.769810:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000001:3.0:1775767854.769812:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767854.769815:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000800:00000200:2.1:1775767854.769815:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000020:00000001:3.0:1775767854.769819:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000800:00000100:2.0:1775767854.769823:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000020:00000001:3.0:1775767854.769824:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000800:00000010:2.0:1775767854.769827:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3300. 00000100:00000040:3.0:1775767854.769830:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767854.769832:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767854.769834:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767854.769835:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767854.769836:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:3.0:1775767854.769838:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767854.769839:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767854.769840:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767854.769843:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767854.769844:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767854.769845:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767854.769847:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775767854.769848:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767854.769849:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767854.769852:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ab800. 00000400:00000200:2.0:1775767854.769854:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000040:3.0:1775767854.769854:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:3.0:1775767854.769861:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767854.769863:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767854.769867:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000100:00000001:3.0:1775767854.769867:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767854.769871:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767854.769874:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767854.769875:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775767854.769879:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 02000000:00000010:3.0:1775767854.769879:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b1000. 00000100:00000001:3.0:1775767854.769886:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767854.769887:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767854.769894:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a8a80 x1862022294243200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767879 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767854.769896:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ab800 x1862022294243456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775767854.769918:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1775767854.769918:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767854.769920:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767854.769926:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a8a80 x1862022294243200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767879 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775767854.769927:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000089df80c0] to pc [ptlrpcd_rcv+-1] req@ffff9942099ab800 x1862022294243456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767854.769957:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767854.769958:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767854.769959:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775767854.769962:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:3.0:1775767854.769964:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775767854.769967:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767854.769968:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775767854.769969:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767854.769970:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767854.769974:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5107 00000400:00000200:2.0:1775767854.769977:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767854.769981:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299780925 00000400:00000200:3.0:1775767854.769981:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767854.769988:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000010:2.0:1775767854.769990:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d6240. 00000100:00080000:3.0:1775767854.769991:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767854.769997:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767854.769998:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1775767854.770000:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767854.770004:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767854.770006:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299780925 00000100:00000001:3.0:1775767854.770007:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000010:2.0:1775767854.770010:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d64e0. 00000020:00000001:3.0:1775767854.770011:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767854.770014:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000100:00000001:2.0:1775767854.770016:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000020:00000001:3.0:1775767854.770017:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767854.770021:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000100:00000001:2.0:1775767854.770022:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1775767854.770026:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:2.0:1775767854.770027:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767854.770031:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:2.0:1775767854.770032:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000040:3.0:1775767854.770037:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00100000:2.0:1775767854.770041:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767854/real 1775767854] req@ffff9942099a8a80 x1862022294243200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767854.770041:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767854.770046:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767854.770050:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767854.770054:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:3.0:1775767854.770059:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767854.770064:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767854.770066:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000001:2.0:1775767854.770068:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:3.0:1775767854.770069:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000001:2.0:1775767854.770072:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:3.0:1775767854.770072:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767854.770075:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000010:2.0:1775767854.770076:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39c60. 00000008:00100000:3.0:1775767854.770078:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000400:00000200:2.0:1775767854.770080:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f220 00000008:00000001:3.0:1775767854.770083:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775767854.770084:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f220. 00000100:00000001:3.0:1775767854.770087:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767854.770089:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000010:3.0:1775767854.770092:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aa680. 00000100:00000200:2.0:1775767854.770096:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a8a80 x1862022294243200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1775767854.770097:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:3.0:1775767854.770103:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767854.770105:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767854.770111:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767854.770113:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000200:2.0:1775767854.770114:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a8a80 x1862022294243200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1775767854.770116:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767854.770123:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767854.770125:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:3.0:1775767854.770125:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0400. 00000100:00000001:2.0:1775767854.770128:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767854.770131:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767854.770133:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767854.770134:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a8a80 x1862022294243200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767854.770140:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aa680 x1862022294243584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775767854.770145:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767854.770153:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a8a80 x1862022294243200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767879 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767854.770153:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767854.770173:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767854.770181:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006cb1597d] to pc [ptlrpcd_rcv+-1] req@ffff9942099aa680 x1862022294243584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767854.770183:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767854.770189:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767854.770192:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767854.770194:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767854.770198:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767854.770198:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767854.770200:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767854.770201:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767854.770203:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767854.770205:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775767854.770207:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775767854.770209:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00080000:3.0:1775767854.770210:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775767854.770211:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1775767854.770214:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000040:2.0:1775767854.770217:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a8a80 x1862022294243200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767879 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767854.770218:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5107 00000400:00000200:3.0:1775767854.770227:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767854.770233:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00100000:2.0:1775767854.770234:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294243200:192.168.202.156@tcp:250:kworker.0 00000100:00080000:3.0:1775767854.770237:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1775767854.770239:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767854.770242:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767854.770243:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1775767854.770246:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767854.770248:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a8a80 x1862022294243200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767879 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775767854.770251:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767854.770255:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767854.770259:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767854.770261:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:3.0:1775767854.770264:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767854.770266:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:3.0:1775767854.770270:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:3.0:1775767854.770275:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000001:2.0:1775767854.770276:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767854.770279:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000040:3.0:1775767854.770280:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 02000000:00000010:2.0:1775767854.770282:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f800. 00000100:00000001:3.0:1775767854.770282:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767854.770286:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000001:2.0:1775767854.770288:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1775767854.770288:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775767854.770292:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767854.770292:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:3.0:1775767854.770295:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000020:00000040:2.0:1775767854.770296:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000100:00000001:3.0:1775767854.770312:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767854.770316:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000001:3.0:1775767854.770317:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000010:2.0:1775767854.770320:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b2c00. 02000000:00000001:3.0:1775767854.770321:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767854.770325:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767854.770327:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:2.0:1775767854.770328:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000008:00100000:3.0:1775767854.770329:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 02000000:00000001:2.0:1775767854.770331:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000008:00000001:3.0:1775767854.770332:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775767854.770335:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a8a80. 00000100:00000001:3.0:1775767854.770335:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767854.770338:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aad80. 00000100:00000001:2.0:1775767854.770340:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000020:00000040:3.0:1775767854.770341:0:13471:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00000100:00000001:2.0:1775767854.770343:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1775767854.770344:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767854.770347:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767854.770348:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767854.770349:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767854.770352:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767854.770354:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767854.770357:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2000. 00000100:00000001:2.0:1775767854.770358:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767854.770360:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767854.770361:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767854.770364:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:3.0:1775767854.770365:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aad80 x1862022294243712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775767854.770367:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767854.770369:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767854.770375:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099aa300 x1862022294243328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767854.770400:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767854.770400:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767854.770403:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767854.770404:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767854.770409:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767854.770411:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1775767854.770411:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000045ccf08] to pc [ptlrpcd_rcv+-1] req@ffff9942099aad80 x1862022294243712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1775767854.770413:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767854.770419:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099aa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294243328:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775767854.770427:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1775767854.770428:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767854.770429:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767854.770431:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767854.770433:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:3.0:1775767854.770434:0:13471:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5117) 02000000:00000010:2.0:1775767854.770437:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f800. 02000000:00000001:2.0:1775767854.770439:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767854.770443:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767854.770445:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39c60. 00000400:00000010:2.0:1775767854.770448:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f220. 00000100:00000200:2.0:1775767854.770451:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294243328, portal 10 00000100:00000001:2.0:1775767854.770452:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767854.770453:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640861440 : -112966068690176 : ffff9942099aa300) 00000100:00000040:2.0:1775767854.770458:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099aa300 x1862022294243328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767879 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767854.770463:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767854.770464:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767854.770465:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775767854.770467:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294243328, offset 0 00000400:00000200:2.0:1775767854.770470:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767854.770473:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767854.770475:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767854.770477:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767854.770479:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767854.770481:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767854.770483:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767854.770485:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225074:225074:256:4294967295] 192.168.202.156@tcp LPNI seq info [225070:225070:8:4294967295] 00000400:00000200:2.0:1775767854.770487:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767854.770492:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767854.770495:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767854.770497:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3300. 00000800:00000200:2.0:1775767854.770500:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775767854.770502:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775767854.770505:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767854.770507:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000100:2.0:1775767854.770509:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767854.770511:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3300. 00000400:00000200:2.0:1775767854.770513:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775767854.770515:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775767854.770517:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775767854.770519:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775767854.770521:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767854.770523:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099aa300 x1862022294243328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767879 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767854.770528:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767854.770530:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099aa300 x1862022294243328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767879 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767854.770535:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767854.770536:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775767854.770538:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767854.770541:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:2.0:1775767854.770542:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767854.770544:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299779603 00000100:00000001:2.0:1775767854.770546:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299780925 00000100:00000001:2.0:1775767854.770548:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299779603 00000100:00000001:2.0:1775767854.770549:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299780925 00000100:00000001:2.0:1775767854.770550:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767854.770552:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767854.770553:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767854.770555:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767854/real 1775767854] req@ffff9942099aa300 x1862022294243328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775767854.770560:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767854.770560:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767854.770561:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39c60. 00000400:00000200:2.0:1775767854.770562:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f220 00000400:00000010:2.0:1775767854.770563:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f220. 00000100:00000001:2.0:1775767854.770565:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767854.770566:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099aa300 x1862022294243328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767854.770571:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099aa300 x1862022294243328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767854.770580:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767854.770581:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767854.770582:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767854.770583:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767854.770585:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099aa300 x1862022294243328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767854.770589:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767854.770592:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099aa300 x1862022294243328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767879 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767854.770596:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767854.770597:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767854.770598:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767854.770599:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767854.770600:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767854.770602:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767854.770603:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767854.770605:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767854.770606:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767854.770609:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099aa300 x1862022294243328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767879 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767854.770613:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099aa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294243328:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775767854.770615:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767854.770617:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ab800 x1862022294243456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767854.770622:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767854.770623:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767854.770625:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767854.770626:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767854.770627:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767854.770629:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294243456:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775767854.770631:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767854.770632:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767854.770633:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767854.770634:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775767854.770636:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1d000. 02000000:00000001:2.0:1775767854.770637:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767854.770639:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767854.770641:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39c60. 00000400:00000010:2.0:1775767854.770642:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f220. 00000100:00000200:2.0:1775767854.770644:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294243456, portal 10 00000100:00000001:2.0:1775767854.770646:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767854.770647:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640866816 : -112966068684800 : ffff9942099ab800) 00000100:00000040:2.0:1775767854.770650:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ab800 x1862022294243456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767879 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767854.770654:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767854.770655:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767854.770657:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775767854.770658:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294243456, offset 0 00000400:00000200:2.0:1775767854.770660:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767854.770662:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767854.770665:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767854.770666:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767854.770668:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767854.770670:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767854.770671:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767854.770673:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225075:225075:256:4294967295] 192.168.202.156@tcp LPNI seq info [225071:225071:8:4294967295] 00000400:00000200:2.0:1775767854.770676:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767854.770680:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767854.770683:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767854.770685:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3300. 00000800:00000200:2.0:1775767854.770687:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775767854.770689:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775767854.770691:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767854.770693:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000100:2.0:1775767854.770696:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767854.770697:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3300. 00000400:00000200:2.0:1775767854.770699:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775767854.770701:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775767854.770702:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775767854.770704:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775767854.770705:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767854.770707:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ab800 x1862022294243456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767879 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767854.770712:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767854.770713:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ab800 x1862022294243456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767879 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767854.770717:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767854.770718:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775767854.770720:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767854.770722:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:2.0:1775767854.770724:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767854.770725:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299779595 00000100:00000001:2.0:1775767854.770727:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299780925 00000100:00000001:2.0:1775767854.770729:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299779595 00000100:00000001:2.0:1775767854.770730:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299780925 00000100:00000001:2.0:1775767854.770731:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767854.770732:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767854.770733:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767854.770735:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767854/real 1775767854] req@ffff9942099ab800 x1862022294243456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775767854.770741:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767854.770742:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767854.770743:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39c60. 00000400:00000200:2.0:1775767854.770744:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f220 00000400:00000010:2.0:1775767854.770745:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f220. 00000100:00000001:2.0:1775767854.770746:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767854.770748:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ab800 x1862022294243456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767854.770752:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ab800 x1862022294243456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767854.770756:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767854.770757:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767854.770758:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767854.770759:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767854.770761:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ab800 x1862022294243456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767854.770765:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767854.770767:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ab800 x1862022294243456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767879 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767854.770771:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767854.770772:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767854.770773:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767854.770774:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767854.770775:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767854.770776:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767854.770778:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767854.770780:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767854.770782:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767854.770784:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ab800 x1862022294243456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767879 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767854.770789:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294243456:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775767854.770792:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767854.770793:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099aa680 x1862022294243584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767854.770797:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767854.770798:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767854.770799:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767854.770800:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767854.770802:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767854.770803:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099aa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294243584:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767854.770805:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767854.770806:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767854.770807:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767854.770808:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775767854.770810:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f400. 02000000:00000001:2.0:1775767854.770811:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767854.770813:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767854.770815:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39c60. 00000400:00000010:2.0:1775767854.770817:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f220. 00000100:00000200:2.0:1775767854.770818:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294243584, portal 4 00000100:00000001:2.0:1775767854.770819:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767854.770820:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640862336 : -112966068689280 : ffff9942099aa680) 00000100:00000040:2.0:1775767854.770824:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099aa680 x1862022294243584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767879 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767854.770828:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767854.770830:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767854.770831:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775767854.770832:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294243584, offset 0 00000400:00000200:2.0:1775767854.770835:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767854.770837:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767854.770839:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767854.770841:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767854.770842:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767854.770844:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767854.770845:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767854.770847:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225076:225076:256:4294967295] 192.168.202.156@tcp LPNI seq info [225072:225072:8:4294967295] 00000400:00000200:2.0:1775767854.770850:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767854.770853:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767854.770856:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767854.770857:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3300. 00000800:00000200:2.0:1775767854.770860:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775767854.770861:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775767854.770864:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767854.770866:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000100:2.0:1775767854.770868:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767854.770869:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3300. 00000400:00000200:2.0:1775767854.770871:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775767854.770873:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775767854.770874:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775767854.770876:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775767854.770877:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767854.770879:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099aa680 x1862022294243584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767879 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767854.770884:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767854.770885:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099aa680 x1862022294243584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767879 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767854.770889:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767854.770890:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775767854.770891:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767854.770893:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:2.0:1775767854.770895:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767854.770897:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299779618 00000100:00000001:2.0:1775767854.770898:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299780925 00000100:00000001:2.0:1775767854.770900:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299779618 00000100:00000001:2.0:1775767854.770901:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299780925 00000100:00000001:2.0:1775767854.770903:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767854.770904:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767854.770905:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767854.770907:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767854/real 1775767854] req@ffff9942099aa680 x1862022294243584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775767854.770911:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767854.770912:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767854.770913:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39c60. 00000400:00000200:2.0:1775767854.770914:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f220 00000400:00000010:2.0:1775767854.770915:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f220. 00000100:00000001:2.0:1775767854.770916:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767854.770918:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099aa680 x1862022294243584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767854.770923:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099aa680 x1862022294243584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767854.770927:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767854.770928:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767854.770929:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767854.770930:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767854.770932:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099aa680 x1862022294243584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767854.770936:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767854.770938:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099aa680 x1862022294243584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767879 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767854.770941:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767854.770943:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767854.770944:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767854.770945:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767854.770946:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767854.770949:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767854.770950:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767854.770952:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767854.770953:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767854.770956:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099aa680 x1862022294243584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767879 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767854.770960:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099aa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294243584:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767854.770962:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767854.770963:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767854.770965:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099aa300 x1862022294243328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767879 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767854.770969:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767854.770970:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767854.770971:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f800. 02000000:00000001:2.0:1775767854.770973:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767854.770974:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767854.770974:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:2.0:1775767854.770976:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767854.770977:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b3400. 02000000:00000001:2.0:1775767854.770978:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767854.770979:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767854.770980:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099aa300. 00000100:00000001:2.0:1775767854.770982:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767854.770983:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767854.770984:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767854.770986:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ab800 x1862022294243456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767879 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767854.770994:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767854.770995:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767854.770996:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1d000. 02000000:00000001:2.0:1775767854.770999:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767854.770999:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767854.771000:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:2.0:1775767854.771001:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767854.771002:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b1000. 02000000:00000001:2.0:1775767854.771003:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767854.771004:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767854.771005:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ab800. 00000100:00000001:2.0:1775767854.771006:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767854.771007:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767854.771008:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767854.771009:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099aa680 x1862022294243584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767879 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767854.771013:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767854.771014:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767854.771015:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f400. 02000000:00000001:2.0:1775767854.771016:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767854.771016:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767854.771017:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:2.0:1775767854.771018:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767854.771019:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b0400. 02000000:00000001:2.0:1775767854.771021:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767854.771021:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767854.771022:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099aa680. 00000100:00000001:2.0:1775767854.771023:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767854.771024:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767854.771025:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767854.771027:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767854.771028:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767854.771029:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775767854.771030:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767854.771030:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767854.771033:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099aad80 x1862022294243712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767854.771037:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767854.771038:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767854.771039:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767854.771040:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767854.771041:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767854.771043:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099aad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294243712:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767854.771045:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767854.771046:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767854.771047:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767854.771048:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775767854.771049:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f400. 02000000:00000001:2.0:1775767854.771050:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767854.771052:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767854.771054:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39c60. 00000400:00000010:2.0:1775767854.771056:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f220. 00000100:00000200:2.0:1775767854.771057:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294243712, portal 4 00000100:00000001:2.0:1775767854.771059:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767854.771060:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640864128 : -112966068687488 : ffff9942099aad80) 00000100:00000040:2.0:1775767854.771063:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099aad80 x1862022294243712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767879 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767854.771067:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767854.771068:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767854.771070:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775767854.771072:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294243712, offset 0 00000400:00000200:2.0:1775767854.771074:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767854.771076:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767854.771078:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767854.771079:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767854.771081:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767854.771082:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767854.771084:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767854.771086:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225077:225077:256:4294967295] 192.168.202.156@tcp LPNI seq info [225073:225073:8:4294967295] 00000400:00000200:2.0:1775767854.771089:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767854.771093:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767854.771095:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767854.771097:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3300. 00000800:00000200:2.0:1775767854.771100:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775767854.771101:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775767854.771104:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767854.771106:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000100:2.0:1775767854.771108:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767854.771109:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3300. 00000400:00000200:2.0:1775767854.771111:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775767854.771113:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775767854.771115:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775767854.771116:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775767854.771117:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767854.771119:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099aad80 x1862022294243712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767879 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767854.771124:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767854.771126:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099aad80 x1862022294243712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767879 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767854.771130:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767854.771131:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775767854.771132:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767854.771134:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:2.0:1775767854.771136:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767854.771137:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299779621 00000100:00000001:2.0:1775767854.771139:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299780925 00000100:00000001:2.0:1775767854.771140:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299779621 00000100:00000001:2.0:1775767854.771142:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299780925 00000100:00000001:2.0:1775767854.771143:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767854.771145:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767854.771146:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767854.771148:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767854/real 1775767854] req@ffff9942099aad80 x1862022294243712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775767854.771153:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767854.771154:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767854.771170:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39c60. 00000400:00000200:2.0:1775767854.771172:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f220 00000400:00000010:2.0:1775767854.771173:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f220. 00000100:00000001:2.0:1775767854.771174:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767854.771176:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099aad80 x1862022294243712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767854.771181:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099aad80 x1862022294243712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767854.771185:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767854.771186:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767854.771187:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767854.771188:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767854.771190:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099aad80 x1862022294243712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767854.771194:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767854.771196:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099aad80 x1862022294243712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767879 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767854.771200:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767854.771201:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767854.771202:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767854.771203:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767854.771204:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767854.771206:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767854.771208:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767854.771211:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767854.771212:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767854.771214:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099aad80 x1862022294243712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767879 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767854.771219:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099aad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294243712:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767854.771221:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767854.771222:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767854.771224:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099aad80 x1862022294243712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767879 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767854.771228:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767854.771229:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767854.771230:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f400. 02000000:00000001:2.0:1775767854.771231:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767854.771232:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767854.771233:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:2.0:1775767854.771234:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767854.771235:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b2000. 02000000:00000001:2.0:1775767854.771236:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767854.771237:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767854.771238:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099aad80. 00000100:00000001:2.0:1775767854.771239:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767854.771240:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767854.771241:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767854.771243:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767854.771244:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767854.771246:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775767854.771247:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767854.833337:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767854.833340:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767854.833346:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775767855.793245:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767855.793254:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767855.793256:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767855.793263:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767855.793268:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767855.793273:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767855.793277:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767855.793280:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767855.793284:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767855.793315:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767855.793322:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767855.793326:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767855.793331:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767855.793334:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767855.793337:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767855.793340:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767855.793342:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767855.793344:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767856.113218:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299780925, at 4299780928 00000100:00000010:1.0:1775767856.113233:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d64e0. 00000100:00000001:1.0:1775767856.817251:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767856.817251:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767856.817257:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767856.817258:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767856.817262:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767856.817262:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767856.817269:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:1.0:1775767856.817270:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767856.817275:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767856.817276:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767856.817291:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767856.817292:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767856.817296:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767856.817299:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767856.817301:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767856.817303:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767856.817305:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767856.817308:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767856.881322:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767856.881326:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767856.881329:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775767857.841235:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767857.841239:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767857.841241:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767857.841244:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767857.841246:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767857.841250:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767857.841254:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775767857.841256:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767857.841258:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767857.841260:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767857.841264:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767857.841264:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767857.841266:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767857.841283:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767857.841287:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767857.841288:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767857.841290:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767857.841291:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767857.841293:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00100000:2.0:1775767857.841293:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767857.841298:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767857.841298:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767857.841301:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767857.841303:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767857.841304:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767857.841307:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767857.841310:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767858.865327:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767858.865327:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767858.865327:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767858.865338:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767858.865338:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767858.865338:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767858.865343:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767858.865343:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767858.865343:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767858.865356:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:2.0:1775767858.865357:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775767858.865357:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767858.865362:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767858.865364:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767858.865364:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767858.865369:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767858.865370:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767858.865370:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767858.865380:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767858.865383:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767858.865385:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767858.865387:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767858.865389:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767858.865392:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767858.865393:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767858.865398:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767858.865402:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767858.929359:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767858.929365:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767858.929370:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775767859.889305:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767859.889311:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767859.889314:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767859.889319:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767859.889324:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767859.889330:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767859.889334:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767859.889345:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767859.889350:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767859.889357:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:1.0:1775767859.889358:0:9316:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775767859.889365:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767859.889370:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767859.889371:0:9316:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775767859.889375:0:9316:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:2.0:1775767859.889377:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00080000:1.0:1775767859.889379:0:9316:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775767859.889385:0:9316:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775767859.889385:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767859.889391:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1775767859.889398:0:9316:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5112 00000100:00000001:2.0:1775767859.889398:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767859.889405:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767859.889408:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767859.889420:0:9316:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767859.889426:0:9316:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775767859.889429:0:9316:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775767859.889434:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767859.889437:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767859.889439:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767859.889441:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775767859.889445:0:9316:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775767859.889446:0:9316:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:1.0:1775767859.889448:0:9316:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775767859.889450:0:9316:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:1.0:1775767859.889452:0:9316:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:1.0:1775767859.889453:0:9316:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:1.0:1775767859.889456:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767859.889457:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767859.889458:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767859.889460:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775767859.889461:0:9316:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:1.0:1775767859.889464:0:9316:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775767859.889465:0:9316:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767859.889468:0:9316:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775767859.889470:0:9316:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767859.889472:0:9316:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767859.889474:0:9316:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775767859.889476:0:9316:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775767859.889484:0:9316:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207888700. 00000020:00000040:1.0:1775767859.889486:0:9316:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:1.0:1775767859.889489:0:9316:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775767859.889490:0:9316:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767859.889493:0:9316:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775767859.889495:0:9316:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775767859.889499:0:9316:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775767859.889507:0:9316:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543e800. 00000100:00000001:1.0:1775767859.889512:0:9316:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767859.889519:0:9316:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207888700 x1862022294243840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775767859.889527:0:9316:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775767859.889531:0:9316:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767859.889535:0:9316:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009f2ba301] to pc [ptlrpcd_rcv+-1] req@ffff994207888700 x1862022294243840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767859.889551:0:9316:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767859.889553:0:9316:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767859.889556:0:9316:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775767859.889557:0:9316:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775767859.889558:0:9316:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775767859.889560:0:9316:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775767859.889562:0:9316:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5112 00000400:00000200:1.0:1775767859.889564:0:9316:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767859.889566:0:9316:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775767859.889568:0:9316:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775767859.889570:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767859.889572:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767859.889574:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767859.889576:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775767859.889579:0:9316:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775767859.889581:0:9316:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:1.0:1775767859.889582:0:9316:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775767859.889585:0:9316:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:1.0:1775767859.889586:0:9316:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:1.0:1775767859.889588:0:9316:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:1.0:1775767859.889590:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767859.889591:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767859.889593:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767859.889594:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775767859.889596:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775767859.889599:0:9316:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00000001:2.0:1775767859.889600:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:1.0:1775767859.889603:0:9316:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775767859.889605:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1775767859.889606:0:9316:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767859.889611:0:9316:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775767859.889616:0:9316:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767859.889617:0:9316:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767859.889620:0:9316:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000040:2.0:1775767859.889623:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207888700 x1862022294243840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1775767859.889625:0:9316:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775767859.889638:0:9316:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767859.889647:0:9316:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775767859.889658:0:9316:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207889f80. 00000100:00000001:2.0:1775767859.889659:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000040:1.0:1775767859.889663:0:9316:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000100:00000001:2.0:1775767859.889663:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767859.889671:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775767859.889677:0:9316:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775767859.889678:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767859.889680:0:9316:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767859.889683:0:9316:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775767859.889683:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767859.889690:0:9316:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775767859.889693:0:9316:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767859.889694:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207888700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294243840:192.168.202.156@tcp:250:kworker.0 02000000:00000010:1.0:1775767859.889698:0:9316:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543c400. 00000100:00000001:2.0:1775767859.889701:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1775767859.889702:0:9316:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767859.889707:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767859.889711:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767859.889712:0:9316:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207889f80 x1862022294243968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1775767859.889717:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1775767859.889724:0:9316:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775767859.889727:0:9316:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775767859.889728:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f400. 02000000:00000001:2.0:1775767859.889732:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767859.889733:0:9316:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ed2274f4] to pc [ptlrpcd_rcv+-1] req@ffff994207889f80 x1862022294243968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1775767859.889743:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767859.889744:0:9316:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767859.889751:0:9316:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:2.0:1775767859.889753:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39c60. 00000100:00080000:1.0:1775767859.889755:0:9316:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775767859.889758:0:9316:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775767859.889759:0:9316:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000010:2.0:1775767859.889761:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f220. 00000100:00000001:1.0:1775767859.889762:0:9316:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775767859.889768:0:9316:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5112 00000400:00000200:1.0:1775767859.889771:0:9316:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000200:2.0:1775767859.889773:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294243840, portal 25 00000400:00000200:1.0:1775767859.889774:0:9316:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775767859.889781:0:9316:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1775767859.889783:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000040:1.0:1775767859.889785:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767859.889787:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606120192 : -112966103431424 : ffff994207888700) 00000100:00000001:1.0:1775767859.889789:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767859.889792:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767859.889797:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775767859.889799:0:9316:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775767859.889800:0:9316:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:1.0:1775767859.889803:0:9316:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000040:2.0:1775767859.889805:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207888700 x1862022294243840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767884 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1775767859.889806:0:9316:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:1.0:1775767859.889812:0:9316:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:1.0:1775767859.889821:0:9316:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:1.0:1775767859.889834:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767859.889843:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1775767859.889845:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767859.889851:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000100:00000040:1.0:1775767859.889855:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000010:2.0:1775767859.889858:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000001:1.0:1775767859.889859:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775767859.889863:0:9316:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00000200:2.0:1775767859.889864:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294243840, offset 0 00000100:00080000:1.0:1775767859.889869:0:9316:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775767859.889873:0:9316:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767859.889875:0:9316:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775767859.889882:0:9316:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1775767859.889882:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 02000000:00000001:1.0:1775767859.889888:0:9316:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767859.889889:0:9316:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775767859.889891:0:9316:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775767859.889893:0:9316:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767859.889895:0:9316:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767859.889901:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000010:1.0:1775767859.889904:0:9316:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff99420788b100. 00000020:00000040:1.0:1775767859.889908:0:9316:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000400:00000200:2.0:1775767859.889912:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 02000000:00000001:1.0:1775767859.889914:0:9316:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775767859.889916:0:9316:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767859.889920:0:9316:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:2.0:1775767859.889920:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:1.0:1775767859.889930:0:9316:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:2.0:1775767859.889930:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 02000000:00000001:1.0:1775767859.889935:0:9316:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767859.889939:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 02000000:00000010:1.0:1775767859.889940:0:9316:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543c800. 00000100:00000001:1.0:1775767859.889948:0:9316:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767859.889948:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00100000:1.0:1775767859.889959:0:9316:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff99420788b100 x1862022294244096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1775767859.889960:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225078:225078:256:4294967295] 192.168.202.156@tcp LPNI seq info [225074:225074:8:4294967295] 00000400:00000200:2.0:1775767859.889981:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767859.889988:0:9316:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775767859.889991:0:9316:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767859.889995:0:9316:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e26cf977] to pc [ptlrpcd_rcv+-1] req@ffff99420788b100 x1862022294244096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1775767859.890011:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000100:00000001:1.0:1775767859.890028:0:9316:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1775767859.890030:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00080000:1.0:1775767859.890032:0:9316:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767859.890036:0:9316:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000010:2.0:1775767859.890036:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3300. 00000100:00000001:1.0:1775767859.890038:0:9316:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775767859.890042:0:9316:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775767859.890044:0:9316:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775767859.890048:0:9316:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5112 00000800:00000200:2.0:1775767859.890051:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000400:00000200:1.0:1775767859.890058:0:9316:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000800:00000200:2.0:1775767859.890058:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000400:00000200:1.0:1775767859.890066:0:9316:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775767859.890068:0:9316:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000800:00000200:2.1:1775767859.890074:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000040:1.0:1775767859.890081:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000800:00000200:2.1:1775767859.890083:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000100:00000001:1.0:1775767859.890090:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767859.890098:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000800:00000100:2.0:1775767859.890098:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767859.890103:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3300. 00000100:00000001:1.0:1775767859.890104:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775767859.890112:0:9316:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775767859.890113:0:9316:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:1.0:1775767859.890114:0:9316:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:2.0:1775767859.890116:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000040:1.0:1775767859.890117:0:9316:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:1.0:1775767859.890118:0:9316:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:1.0:1775767859.890120:0:9316:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:1.0:1775767859.890122:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767859.890124:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767859.890125:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767859.890127:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775767859.890131:0:9316:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:1.0:1775767859.890138:0:9316:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000400:00000200:2.0:1775767859.890138:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000100:00000001:1.0:1775767859.890142:0:9316:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767859.890143:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 02000000:00000001:1.0:1775767859.890144:0:9316:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:2.0:1775767859.890150:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 02000000:00000001:1.0:1775767859.890152:0:9316:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767859.890183:0:2399:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:1.0:1775767859.890192:0:9316:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1775767859.890193:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207888700 x1862022294243840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767884 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767859.890201:0:9316:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775767859.890206:0:9316:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:1.0:1775767859.890217:0:9316:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767859.890220:0:9316:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767859.890223:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000010:1.0:1775767859.890224:0:9316:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207888e00. 00000100:00000040:2.0:1775767859.890229:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207888700 x1862022294243840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767884 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1775767859.890230:0:9316:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:1.0:1775767859.890241:0:9316:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775767859.890246:0:9316:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767859.890262:0:9316:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1775767859.890262:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767859.890268:0:9316:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1775767859.890268:0:2399:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:1.0:1775767859.890273:0:9316:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767859.890278:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775767859.890281:0:9316:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543fc00. 00000400:00000200:2.0:1775767859.890287:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767859.890289:0:9316:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767859.890291:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299786046 00000100:00000010:2.0:1775767859.890300:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d6c60. 00000100:00100000:1.0:1775767859.890301:0:9316:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207888e00 x1862022294244224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775767859.890311:0:9316:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775767859.890313:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775767859.890314:0:9316:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767859.890325:0:9316:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001812f966] to pc [ptlrpcd_rcv+-1] req@ffff994207888e00 x1862022294244224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767859.890325:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299786046 00000100:00000010:2.0:1775767859.890331:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d6000. 00000100:00000001:2.0:1775767859.890341:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775767859.890348:0:9316:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767859.890348:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767859.890357:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767859.890358:0:9316:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775767859.890366:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00080000:1.0:1775767859.890367:0:9316:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775767859.890369:0:9316:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775767859.890371:0:9316:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775767859.890377:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767859/real 1775767859] req@ffff994207888700 x1862022294243840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767859.890378:0:9316:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775767859.890388:0:9316:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5112 00000400:00000200:1.0:1775767859.890406:0:9316:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767859.890417:0:9316:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775767859.890428:0:9316:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000400:00000001:2.0:1775767859.890429:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:1.0:1775767859.890432:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000001:2.0:1775767859.890432:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767859.890437:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39c60. 00000100:00000001:1.0:1775767859.890438:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767859.890448:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000200:2.0:1775767859.890448:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f220 00000100:00000001:1.0:1775767859.890454:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000010:2.0:1775767859.890454:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f220. 00000020:00000001:1.0:1775767859.890461:0:9316:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:2.0:1775767859.890462:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000040:1.0:1775767859.890464:0:9316:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:1.0:1775767859.890471:0:9316:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000200:2.0:1775767859.890471:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207888700 x1862022294243840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1775767859.890476:0:9316:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:1.0:1775767859.890485:0:9316:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:1.0:1775767859.890492:0:9316:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000200:2.0:1775767859.890501:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207888700 x1862022294243840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775767859.890504:0:9316:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767859.890511:0:9316:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767859.890521:0:9316:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767859.890529:0:9316:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775767859.890540:0:9316:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00000001:2.0:1775767859.890542:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00080000:1.0:1775767859.890545:0:9316:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775767859.890545:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767859.890552:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767859.890554:0:9316:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767859.890555:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1775767859.890558:0:9316:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:2.0:1775767859.890561:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207888700 x1862022294243840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1775767859.890562:0:9316:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767859.890571:0:9316:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767859.890581:0:9316:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775767859.890585:0:9316:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000100:00000001:2.0:1775767859.890587:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:1.0:1775767859.890593:0:9316:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767859.890596:0:9316:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767859.890599:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207888700 x1862022294243840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767884 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1775767859.890602:0:9316:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207889880. 00000020:00000040:1.0:1775767859.890612:0:9316:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00000100:00000001:2.0:1775767859.890617:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:1.0:1775767859.890618:0:9316:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775767859.890622:0:9316:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767859.890622:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767859.890629:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767859.890630:0:9316:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1775767859.890634:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1775767859.890635:0:9316:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1775767859.890641:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 02000000:00000001:1.0:1775767859.890642:0:9316:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767859.890645:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000010:1.0:1775767859.890647:0:9316:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543f800. 00000100:00080000:2.0:1775767859.890649:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:1.0:1775767859.890652:0:9316:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775767859.890658:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767859.890662:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1775767859.890664:0:9316:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207889880 x1862022294244352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:2.0:1775767859.890674:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207888700 x1862022294243840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767884 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767859.890709:0:9316:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775767859.890717:0:9316:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767859.890723:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207888700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294243840:192.168.202.156@tcp:250:kworker.0 00000100:00000040:1.0:1775767859.890729:0:9316:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f11856b8] to pc [ptlrpcd_rcv+-1] req@ffff994207889880 x1862022294244352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767859.890740:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767859.890751:0:9316:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767859.890751:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767859.890760:0:9316:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5122) 00000100:00000040:2.0:1775767859.890761:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207888700 x1862022294243840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767884 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767859.890776:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767859.890781:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767859.890785:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f400. 02000000:00000001:2.0:1775767859.890789:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767859.890792:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767859.890793:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:2.0:1775767859.890796:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767859.890799:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543e800. 02000000:00000001:2.0:1775767859.890803:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767859.890805:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767859.890807:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207888700. 00000100:00000001:2.0:1775767859.890811:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767859.890812:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767859.890815:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767859.890823:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767859.890825:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767859.890827:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775767859.890830:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767859.890831:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767859.890835:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207889f80 x1862022294243968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767859.890841:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767859.890843:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767859.890844:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767859.890847:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767859.890848:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767859.890850:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207889f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294243968:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775767859.890853:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767859.890854:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767859.890856:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767859.890857:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775767859.890859:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f400. 02000000:00000001:2.0:1775767859.890861:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767859.890865:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767859.890867:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39c60. 00000400:00000010:2.0:1775767859.890870:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f220. 00000100:00000200:2.0:1775767859.890872:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294243968, portal 10 00000100:00000001:2.0:1775767859.890874:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767859.890875:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606126464 : -112966103425152 : ffff994207889f80) 00000100:00000040:2.0:1775767859.890880:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207889f80 x1862022294243968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767884 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767859.890885:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767859.890887:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767859.890888:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775767859.890890:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294243968, offset 0 00000400:00000200:2.0:1775767859.890893:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767859.890896:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767859.890899:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767859.890901:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767859.890903:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767859.890905:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767859.890906:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767859.890909:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225079:225079:256:4294967295] 192.168.202.156@tcp LPNI seq info [225075:225075:8:4294967295] 00000400:00000200:2.0:1775767859.890912:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767859.890916:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767859.890919:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767859.890921:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3300. 00000800:00000200:2.0:1775767859.890924:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775767859.890926:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775767859.890930:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767859.890931:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000100:2.0:1775767859.890934:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767859.890935:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3300. 00000400:00000200:2.0:1775767859.890937:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775767859.890940:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775767859.890942:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775767859.890943:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775767859.890945:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767859.890947:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207889f80 x1862022294243968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767884 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767859.890953:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767859.890955:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207889f80 x1862022294243968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767884 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767859.890959:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767859.890960:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775767859.890962:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767859.890965:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:2.0:1775767859.890968:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767859.890970:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299784723 00000100:00000001:2.0:1775767859.890972:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299786046 00000100:00000001:2.0:1775767859.890974:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299784723 00000100:00000001:2.0:1775767859.890975:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299786046 00000100:00000001:2.0:1775767859.890977:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767859.890978:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767859.890979:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767859.890981:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767859/real 1775767859] req@ffff994207889f80 x1862022294243968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775767859.890986:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767859.890987:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767859.890988:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39c60. 00000400:00000200:2.0:1775767859.890990:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f220 00000400:00000010:2.0:1775767859.890991:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f220. 00000100:00000001:2.0:1775767859.890993:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767859.890994:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207889f80 x1862022294243968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767859.891000:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207889f80 x1862022294243968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767859.891003:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767859.891004:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767859.891005:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767859.891006:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767859.891008:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207889f80 x1862022294243968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767859.891012:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767859.891014:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207889f80 x1862022294243968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767884 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767859.891018:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767859.891020:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767859.891021:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767859.891022:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767859.891023:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767859.891025:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767859.891027:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767859.891030:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767859.891031:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767859.891034:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207889f80 x1862022294243968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767884 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767859.891040:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207889f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294243968:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775767859.891043:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767859.891045:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff99420788b100 x1862022294244096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767859.891050:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767859.891051:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767859.891053:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767859.891054:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767859.891055:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767859.891056:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff99420788b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294244096:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775767859.891058:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767859.891059:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767859.891060:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767859.891062:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775767859.891064:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1d000. 02000000:00000001:2.0:1775767859.891066:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767859.891068:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767859.891069:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39c60. 00000400:00000010:2.0:1775767859.891071:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f220. 00000100:00000200:2.0:1775767859.891073:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294244096, portal 10 00000100:00000001:2.0:1775767859.891074:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767859.891075:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606130944 : -112966103420672 : ffff99420788b100) 00000100:00000040:2.0:1775767859.891079:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff99420788b100 x1862022294244096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767884 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767859.891084:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767859.891086:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767859.891087:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775767859.891089:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294244096, offset 0 00000400:00000200:2.0:1775767859.891091:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767859.891093:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767859.891095:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767859.891097:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767859.891099:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767859.891100:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767859.891102:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767859.891104:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225080:225080:256:4294967295] 192.168.202.156@tcp LPNI seq info [225076:225076:8:4294967295] 00000400:00000200:2.0:1775767859.891107:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767859.891111:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767859.891114:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767859.891115:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3300. 00000800:00000200:2.0:1775767859.891118:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775767859.891119:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775767859.891122:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767859.891124:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000100:2.0:1775767859.891127:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767859.891128:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3300. 00000400:00000200:2.0:1775767859.891130:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775767859.891132:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775767859.891133:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775767859.891134:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775767859.891136:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767859.891138:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff99420788b100 x1862022294244096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767884 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767859.891143:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767859.891145:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff99420788b100 x1862022294244096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767884 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767859.891149:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767859.891150:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775767859.891151:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767859.891153:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:2.0:1775767859.891174:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767859.891176:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299784716 00000100:00000001:2.0:1775767859.891177:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299786046 00000100:00000001:2.0:1775767859.891178:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299784716 00000100:00000001:2.0:1775767859.891181:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299786046 00000100:00000001:2.0:1775767859.891182:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767859.891184:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767859.891185:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767859.891188:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767859/real 1775767859] req@ffff99420788b100 x1862022294244096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775767859.891193:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767859.891194:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767859.891195:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39c60. 00000400:00000200:2.0:1775767859.891197:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f220 00000400:00000010:2.0:1775767859.891198:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f220. 00000100:00000001:2.0:1775767859.891199:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767859.891201:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff99420788b100 x1862022294244096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767859.891207:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff99420788b100 x1862022294244096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767859.891211:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767859.891212:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767859.891213:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767859.891214:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767859.891216:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff99420788b100 x1862022294244096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767859.891219:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767859.891221:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff99420788b100 x1862022294244096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767884 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767859.891225:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767859.891227:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767859.891228:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767859.891229:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767859.891230:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767859.891233:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767859.891235:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767859.891237:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767859.891239:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767859.891242:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff99420788b100 x1862022294244096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767884 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767859.891247:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff99420788b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294244096:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775767859.891262:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767859.891265:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207888e00 x1862022294244224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767859.891270:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767859.891271:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767859.891273:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767859.891274:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767859.891276:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767859.891278:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207888e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294244224:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767859.891280:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767859.891282:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767859.891283:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767859.891284:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775767859.891286:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f800. 02000000:00000001:2.0:1775767859.891287:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767859.891290:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767859.891292:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39c60. 00000400:00000010:2.0:1775767859.891294:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f220. 00000100:00000200:2.0:1775767859.891297:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294244224, portal 4 00000100:00000001:2.0:1775767859.891298:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767859.891299:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606121984 : -112966103429632 : ffff994207888e00) 00000100:00000040:2.0:1775767859.891303:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207888e00 x1862022294244224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767884 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767859.891308:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767859.891310:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767859.891311:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775767859.891313:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294244224, offset 0 00000400:00000200:2.0:1775767859.891315:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767859.891318:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767859.891320:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767859.891322:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767859.891324:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767859.891326:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767859.891327:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767859.891329:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225081:225081:256:4294967295] 192.168.202.156@tcp LPNI seq info [225077:225077:8:4294967295] 00000400:00000200:2.0:1775767859.891332:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767859.891336:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767859.891339:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767859.891341:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3300. 00000800:00000200:2.0:1775767859.891344:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775767859.891346:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775767859.891349:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767859.891351:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000100:2.0:1775767859.891354:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767859.891355:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3300. 00000400:00000200:2.0:1775767859.891357:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775767859.891360:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775767859.891362:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775767859.891363:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775767859.891365:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767859.891367:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207888e00 x1862022294244224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767884 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767859.891372:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767859.891374:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207888e00 x1862022294244224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767884 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767859.891383:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767859.891384:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775767859.891386:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767859.891389:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:2.0:1775767859.891391:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767859.891392:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299784739 00000100:00000001:2.0:1775767859.891394:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299786046 00000100:00000001:2.0:1775767859.891396:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299784739 00000100:00000001:2.0:1775767859.891397:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299786046 00000100:00000001:2.0:1775767859.891398:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767859.891399:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767859.891401:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767859.891403:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767859/real 1775767859] req@ffff994207888e00 x1862022294244224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775767859.891407:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767859.891408:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767859.891409:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39c60. 00000400:00000200:2.0:1775767859.891411:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f220 00000400:00000010:2.0:1775767859.891412:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f220. 00000100:00000001:2.0:1775767859.891413:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767859.891415:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207888e00 x1862022294244224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767859.891420:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207888e00 x1862022294244224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767859.891424:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767859.891424:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767859.891425:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767859.891426:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767859.891428:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207888e00 x1862022294244224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767859.891433:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767859.891436:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207888e00 x1862022294244224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767884 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767859.891440:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767859.891441:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767859.891442:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767859.891443:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767859.891444:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767859.891447:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767859.891448:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767859.891451:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767859.891452:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767859.891454:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207888e00 x1862022294244224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767884 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767859.891458:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207888e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294244224:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767859.891461:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767859.891463:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207889880 x1862022294244352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767859.891468:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767859.891469:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767859.891470:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767859.891471:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767859.891472:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767859.891474:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207889880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294244352:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767859.891476:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767859.891477:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767859.891478:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767859.891479:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775767859.891482:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1e400. 02000000:00000001:2.0:1775767859.891483:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767859.891485:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767859.891486:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39c60. 00000400:00000010:2.0:1775767859.891488:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f220. 00000100:00000200:2.0:1775767859.891490:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294244352, portal 4 00000100:00000001:2.0:1775767859.891491:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767859.891492:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606124672 : -112966103426944 : ffff994207889880) 00000100:00000040:2.0:1775767859.891495:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207889880 x1862022294244352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767884 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767859.891500:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767859.891501:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767859.891503:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775767859.891504:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294244352, offset 0 00000400:00000200:2.0:1775767859.891507:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767859.891508:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767859.891510:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767859.891512:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767859.891513:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767859.891514:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767859.891516:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767859.891518:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225082:225082:256:4294967295] 192.168.202.156@tcp LPNI seq info [225078:225078:8:4294967295] 00000400:00000200:2.0:1775767859.891521:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767859.891525:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767859.891527:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767859.891529:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3300. 00000800:00000200:2.0:1775767859.891531:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775767859.891532:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775767859.891535:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767859.891537:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000100:2.0:1775767859.891540:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767859.891541:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3300. 00000400:00000200:2.0:1775767859.891543:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775767859.891545:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775767859.891547:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775767859.891548:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775767859.891549:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767859.891551:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207889880 x1862022294244352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767884 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767859.891556:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767859.891559:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207889880 x1862022294244352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767884 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767859.891563:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767859.891564:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775767859.891566:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767859.891567:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:2.0:1775767859.891569:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767859.891570:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299784742 00000100:00000001:2.0:1775767859.891572:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299786046 00000100:00000001:2.0:1775767859.891574:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299784742 00000100:00000001:2.0:1775767859.891575:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299786046 00000100:00000001:2.0:1775767859.891576:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767859.891577:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767859.891579:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767859.891580:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767859/real 1775767859] req@ffff994207889880 x1862022294244352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775767859.891585:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767859.891586:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767859.891586:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39c60. 00000400:00000200:2.0:1775767859.891588:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f220 00000400:00000010:2.0:1775767859.891589:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f220. 00000100:00000001:2.0:1775767859.891590:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767859.891592:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207889880 x1862022294244352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767859.891598:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207889880 x1862022294244352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767859.891601:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767859.891602:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767859.891603:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767859.891604:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767859.891606:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207889880 x1862022294244352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767859.891610:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767859.891612:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207889880 x1862022294244352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767884 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767859.891616:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767859.891617:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767859.891618:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767859.891619:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767859.891620:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767859.891623:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767859.891624:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767859.891627:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767859.891628:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767859.891630:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207889880 x1862022294244352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767884 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767859.891634:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207889880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294244352:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767859.891637:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767859.891638:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767859.891746:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207889f80 x1862022294243968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767884 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767859.891755:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767859.891757:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767859.891760:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f400. 02000000:00000001:2.0:1775767859.891764:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767859.891765:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767859.891766:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:2.0:1775767859.891768:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767859.891769:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543c400. 02000000:00000001:2.0:1775767859.891772:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767859.891773:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767859.891775:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207889f80. 00000100:00000001:2.0:1775767859.891778:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767859.891779:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767859.891781:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767859.891784:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff99420788b100 x1862022294244096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767884 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767859.891789:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767859.891790:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767859.891791:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1d000. 02000000:00000001:2.0:1775767859.891793:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767859.891794:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767859.891795:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:2.0:1775767859.891796:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767859.891797:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543c800. 02000000:00000001:2.0:1775767859.891798:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767859.891799:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767859.891799:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff99420788b100. 00000100:00000001:2.0:1775767859.891801:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767859.891802:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767859.891803:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767859.891805:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207888e00 x1862022294244224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767884 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767859.891809:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767859.891811:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767859.891811:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f800. 02000000:00000001:2.0:1775767859.891813:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767859.891814:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767859.891815:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:2.0:1775767859.891816:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767859.891817:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543fc00. 02000000:00000001:2.0:1775767859.891818:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767859.891819:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767859.891820:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207888e00. 00000100:00000001:2.0:1775767859.891821:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767859.891822:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767859.891823:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767859.891825:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207889880 x1862022294244352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767884 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767859.891829:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767859.891830:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767859.891831:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1e400. 02000000:00000001:2.0:1775767859.891832:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767859.891833:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767859.891834:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:2.0:1775767859.891835:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767859.891836:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543f800. 02000000:00000001:2.0:1775767859.891837:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767859.891838:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767859.891838:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207889880. 00000100:00000001:2.0:1775767859.891839:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767859.891840:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767859.891841:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767859.891847:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767859.891848:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767859.891849:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775767859.891850:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767860.913264:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767860.913268:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767860.913270:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767860.913273:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767860.913274:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767860.913276:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767860.913279:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775767860.913282:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767860.913284:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767860.913288:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767860.913290:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767860.913291:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767860.913298:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767860.913300:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767860.913302:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767860.913303:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767860.913306:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767860.913308:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767860.977360:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767860.977365:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767860.977368:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775767861.233248:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299786046, at 4299786048 00000100:00000010:1.0:1775767861.233265:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d6000. 00000100:00000001:2.0:1775767861.937268:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767861.937268:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767861.937274:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767861.937275:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767861.937276:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767861.937278:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767861.937286:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775767861.937287:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767861.937291:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767861.937293:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767861.937296:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767861.937296:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767861.937300:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767861.937303:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767861.937305:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767861.937308:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767861.937311:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767861.937314:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767862.961308:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767862.961313:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767862.961316:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767862.961322:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767862.961324:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767862.961331:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767862.961334:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775767862.961342:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775767862.961344:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767862.961349:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767862.961353:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767862.961358:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767862.961361:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767862.961479:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767862.961483:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767862.961485:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767862.961487:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767862.961490:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767863.025372:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767863.025377:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767863.025381:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775767863.985256:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767863.985265:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767863.985273:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767863.985278:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:2.0:1775767863.985282:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767863.985284:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767863.985289:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767863.985290:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767863.985299:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767863.985299:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775767863.985312:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767863.985312:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767863.985318:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767863.985322:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767863.985331:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767863.985345:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767863.985349:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767863.985353:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767865.009266:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767865.009273:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767865.009277:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767865.009284:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767865.009290:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767865.009295:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767865.009307:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767865.009310:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767865.009314:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767865.009320:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767865.009327:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775767865.009330:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767865.009333:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767865.009336:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767865.009344:0:13471:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5117 00000400:00000200:3.0:1775767865.009355:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767865.009367:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767865.009371:0:13471:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767865.009376:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767865.009381:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767865.009385:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767865.009388:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767865.009392:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767865.009395:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:3.0:1775767865.009397:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767865.009399:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:3.0:1775767865.009401:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:3.0:1775767865.009403:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:3.0:1775767865.009405:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767865.009406:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767865.009408:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767865.009409:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767865.009411:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:3.0:1775767865.009413:0:13471:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767865.009415:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767865.009417:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767865.009419:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767865.009421:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767865.009422:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775767865.009424:0:13471:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767865.009438:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aad80. 00000020:00000040:3.0:1775767865.009441:0:13471:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:3.0:1775767865.009444:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767865.009445:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767865.009447:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767865.009449:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767865.009453:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767865.009461:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0c00. 00000100:00000001:3.0:1775767865.009466:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767865.009473:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aad80 x1862022294244480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767865.009481:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767865.009484:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767865.009488:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000045ccf08] to pc [ptlrpcd_rcv+-1] req@ffff9942099aad80 x1862022294244480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767865.009502:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767865.009505:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767865.009508:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775767865.009509:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767865.009511:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767865.009513:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767865.009515:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5117 00000400:00000200:3.0:1775767865.009518:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767865.009520:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767865.009522:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767865.009524:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767865.009525:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767865.009527:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767865.009528:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767865.009529:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767865.009530:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:3.0:1775767865.009532:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767865.009533:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:3.0:1775767865.009535:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:3.0:1775767865.009536:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:3.0:1775767865.009539:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767865.009541:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775767865.009542:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1775767865.009543:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767865.009546:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767865.009546:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:2.0:1775767865.009549:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000040:3.0:1775767865.009550:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00080000:3.0:1775767865.009554:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000040:2.0:1775767865.009558:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099aad80 x1862022294244480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767865.009558:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767865.009560:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767865.009565:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767865.009568:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1775767865.009568:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767865.009569:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767865.009571:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:2.0:1775767865.009574:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:3.0:1775767865.009575:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1775767865.009577:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:3.0:1775767865.009578:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767865.009580:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767865.009581:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767865.009586:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aa680. 00000100:00100000:2.0:1775767865.009587:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099aad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294244480:192.168.202.156@tcp:250:kworker.0 00000020:00000040:3.0:1775767865.009589:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000100:00000001:2.0:1775767865.009593:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767865.009594:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767865.009596:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767865.009598:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1775767865.009598:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767865.009599:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775767865.009601:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767865.009602:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767865.009605:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767865.009606:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2800. 00000100:00000001:3.0:1775767865.009609:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775767865.009610:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1e400. 00000100:00100000:3.0:1775767865.009614:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aa680 x1862022294244608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1775767865.009615:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767865.009625:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767865.009626:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000010:2.0:1775767865.009629:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39c60. 00000100:00000001:3.0:1775767865.009629:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767865.009635:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006cb1597d] to pc [ptlrpcd_rcv+-1] req@ffff9942099aa680 x1862022294244608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1775767865.009636:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f220. 00000100:00000001:3.0:1775767865.009644:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1775767865.009645:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294244480, portal 25 00000100:00080000:3.0:1775767865.009646:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775767865.009649:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1775767865.009651:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775767865.009652:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640864128 : -112966068687488 : ffff9942099aad80) 00000100:00000001:3.0:1775767865.009654:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767865.009656:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:2.0:1775767865.009660:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099aad80 x1862022294244480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767890 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767865.009661:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767865.009665:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5117 00000400:00000200:3.0:1775767865.009673:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767865.009677:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767865.009682:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1775767865.009688:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767865.009688:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767865.009690:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767865.009694:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000100:00000040:3.0:1775767865.009696:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000010:2.0:1775767865.009697:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000001:3.0:1775767865.009698:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000200:2.0:1775767865.009702:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294244480, offset 0 00000020:00000001:3.0:1775767865.009702:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767865.009704:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:3.0:1775767865.009706:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767865.009708:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000400:00000200:2.0:1775767865.009711:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000020:00000001:3.0:1775767865.009712:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:3.0:1775767865.009715:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:3.0:1775767865.009718:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000200:2.0:1775767865.009722:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767865.009722:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767865.009726:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767865.009728:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000200:2.0:1775767865.009730:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000020:00000040:3.0:1775767865.009731:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:3.0:1775767865.009737:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000400:00000200:2.0:1775767865.009738:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1775767865.009740:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767865.009742:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:2.0:1775767865.009744:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 02000000:00000001:3.0:1775767865.009746:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1775767865.009749:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 02000000:00000001:3.0:1775767865.009750:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767865.009753:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:2.0:1775767865.009754:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000008:00100000:3.0:1775767865.009755:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775767865.009758:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767865.009761:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225083:225083:256:4294967295] 192.168.202.156@tcp LPNI seq info [225079:225079:8:4294967295] 00000100:00000001:3.0:1775767865.009761:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767865.009765:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ab800. 00000400:00000200:2.0:1775767865.009769:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000040:3.0:1775767865.009770:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:3.0:1775767865.009773:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767865.009774:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767865.009776:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767865.009778:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767865.009781:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767865.009783:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 02000000:00000010:3.0:1775767865.009785:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0000. 00000100:00000001:3.0:1775767865.009789:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1775767865.009790:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00100000:3.0:1775767865.009794:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ab800 x1862022294244736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000010:2.0:1775767865.009795:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3300. 00000100:00000001:3.0:1775767865.009803:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000200:2.0:1775767865.009806:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:3.0:1775767865.009806:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1775767865.009812:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000100:00000040:3.0:1775767865.009813:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000089df80c0] to pc [ptlrpcd_rcv+-1] req@ffff9942099ab800 x1862022294244736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.1:1775767865.009823:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:3.0:1775767865.009826:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775767865.009827:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000100:00080000:3.0:1775767865.009829:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1775767865.009834:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000100:00080000:3.0:1775767865.009835:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000010:2.0:1775767865.009838:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3300. 00000100:00000001:3.0:1775767865.009838:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767865.009841:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767865.009843:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:2.0:1775767865.009844:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00080000:3.0:1775767865.009846:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5117 00000400:00000200:2.0:1775767865.009851:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767865.009854:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767865.009856:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000200:3.0:1775767865.009859:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000400:00000010:2.0:1775767865.009860:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00080000:3.0:1775767865.009861:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1775767865.009865:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:3.0:1775767865.009865:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767865.009868:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:2.0:1775767865.009872:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099aad80 x1862022294244480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767890 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775767865.009872:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767865.009877:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767865.009881:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767865.009882:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:3.0:1775767865.009883:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767865.009885:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:3.0:1775767865.009888:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:3.0:1775767865.009893:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:2.0:1775767865.009896:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767865.009900:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000040:2.0:1775767865.009902:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099aad80 x1862022294244480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767890 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767865.009904:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767865.009910:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767865.009915:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767865.009922:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00000001:2.0:1775767865.009923:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767865.009926:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:3.0:1775767865.009929:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775767865.009931:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767865.009933:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767865.009937:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:3.0:1775767865.009939:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1775767865.009940:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299791165 02000000:00000001:3.0:1775767865.009944:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000010:2.0:1775767865.009946:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d6360. 02000000:00000001:3.0:1775767865.009948:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767865.009951:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767865.009953:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000100:00000001:2.0:1775767865.009954:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000008:00000001:3.0:1775767865.009958:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767865.009960:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767865.009962:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299791165 00000100:00000010:3.0:1775767865.009964:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aa300. 00000100:00000010:2.0:1775767865.009966:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d6660. 00000020:00000040:3.0:1775767865.009967:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00000100:00000001:2.0:1775767865.009969:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 02000000:00000001:3.0:1775767865.009970:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767865.009973:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767865.009974:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767865.009974:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767865.009976:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767865.009978:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767865.009982:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767865.009983:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2000. 00000100:00000001:2.0:1775767865.009984:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1775767865.009987:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767865.009990:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767865/real 1775767865] req@ffff9942099aad80 x1862022294244480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767865.009994:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aa300 x1862022294244864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:2.0:1775767865.010009:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1775767865.010010:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000001:2.0:1775767865.010011:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767865.010014:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39c60. 00000100:00000001:3.0:1775767865.010014:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767865.010018:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f220 00000100:00000040:3.0:1775767865.010020:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f33bdf15] to pc [ptlrpcd_rcv+-1] req@ffff9942099aa300 x1862022294244864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1775767865.010021:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f220. 00000100:00000001:2.0:1775767865.010025:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767865.010032:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099aad80 x1862022294244480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767865.010032:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767865.010037:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767865.010045:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775767865.010047:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767865.010050:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:2.0:1775767865.010052:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099aad80 x1862022294244480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767865.010053:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767865.010058:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5117 00000100:00000001:2.0:1775767865.010067:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:3.0:1775767865.010067:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767865.010070:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:2.0:1775767865.010071:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775767865.010074:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1775767865.010075:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767865.010078:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767865.010079:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767865.010082:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:2.0:1775767865.010085:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099aad80 x1862022294244480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775767865.010087:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767865.010092:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767865.010096:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767865.010099:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:3.0:1775767865.010102:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:2.0:1775767865.010103:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1775767865.010105:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000100:00000040:2.0:1775767865.010109:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099aad80 x1862022294244480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767890 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775767865.010110:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:3.0:1775767865.010116:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:3.0:1775767865.010122:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767865.010128:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775767865.010129:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767865.010134:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767865.010134:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767865.010137:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767865.010140:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767865.010140:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:2.0:1775767865.010144:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000020:00000040:3.0:1775767865.010145:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00000001:2.0:1775767865.010146:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767865.010150:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767865.010152:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00080000:2.0:1775767865.010175:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767865.010177:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767865.010187:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099aad80 x1862022294244480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767890 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767865.010193:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099aad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294244480:192.168.202.156@tcp:250:kworker.0 00000100:00000001:2.0:1775767865.010211:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767865.010213:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767865.010215:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099aad80 x1862022294244480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767890 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767865.010220:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767865.010223:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767865.010225:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1e400. 02000000:00000001:2.0:1775767865.010228:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767865.010229:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767865.010231:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:2.0:1775767865.010232:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767865.010234:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b0c00. 00000100:00000001:3.0:1775767865.010239:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767865.010241:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767865.010241:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775767865.010245:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:3.0:1775767865.010246:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000010:2.0:1775767865.010247:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099aad80. 02000000:00000001:3.0:1775767865.010249:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767865.010251:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775767865.010252:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000008:00100000:3.0:1775767865.010253:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000100:00000001:2.0:1775767865.010254:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767865.010258:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1775767865.010260:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767865.010263:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767865.010264:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767865.010266:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767865.010267:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8a80. 00000100:00000001:2.0:1775767865.010268:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775767865.010270:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767865.010272:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000040:3.0:1775767865.010273:0:13471:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00000100:00000040:2.0:1775767865.010276:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099aa680 x1862022294244608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1775767865.010280:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767865.010283:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767865.010286:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767865.010287:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1775767865.010288:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767865.010291:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767865.010291:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775767865.010295:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767865.010296:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767865.010297:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767865.010302:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099aa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294244608:192.168.202.156@tcp:38:kworker.0 02000000:00000010:3.0:1775767865.010304:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0400. 00000100:00000001:2.0:1775767865.010305:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767865.010307:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767865.010309:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767865.010311:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1775767865.010312:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775767865.010315:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1e400. 02000000:00000001:2.0:1775767865.010317:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767865.010322:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8a80 x1862022294244992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1775767865.010323:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767865.010326:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39c60. 00000400:00000010:2.0:1775767865.010331:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f220. 00000100:00000200:2.0:1775767865.010336:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294244608, portal 10 00000100:00000001:2.0:1775767865.010340:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767865.010341:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775767865.010342:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640862336 : -112966068689280 : ffff9942099aa680) 00000100:00000001:3.0:1775767865.010346:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767865.010350:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099aa680 x1862022294244608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767890 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775767865.010356:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006c3265af] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8a80 x1862022294244992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767865.010359:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767865.010365:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767865.010369:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775767865.010373:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294244608, offset 0 00000100:00000001:3.0:1775767865.010378:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767865.010379:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767865.010384:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000040:3.0:1775767865.010386:0:13471:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5127) 00000400:00000200:2.0:1775767865.010388:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767865.010393:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767865.010396:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767865.010398:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767865.010399:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767865.010402:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225084:225084:256:4294967295] 192.168.202.156@tcp LPNI seq info [225080:225080:8:4294967295] 00000400:00000200:2.0:1775767865.010406:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767865.010414:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767865.010419:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767865.010422:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3300. 00000800:00000200:2.0:1775767865.010426:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775767865.010429:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775767865.010434:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767865.010436:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000100:2.0:1775767865.010440:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767865.010442:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3300. 00000400:00000200:2.0:1775767865.010445:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775767865.010449:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775767865.010452:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775767865.010454:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775767865.010457:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767865.010461:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099aa680 x1862022294244608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767890 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767865.010469:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767865.010471:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099aa680 x1862022294244608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767890 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767865.010476:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767865.010477:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775767865.010480:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767865.010483:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:2.0:1775767865.010485:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767865.010487:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299789843 00000100:00000001:2.0:1775767865.010489:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299791165 00000100:00000001:2.0:1775767865.010490:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299789843 00000100:00000001:2.0:1775767865.010492:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299791165 00000100:00000001:2.0:1775767865.010494:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767865.010495:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767865.010496:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767865.010499:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767865/real 1775767865] req@ffff9942099aa680 x1862022294244608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775767865.010503:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767865.010504:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767865.010505:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39c60. 00000400:00000200:2.0:1775767865.010507:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f220 00000400:00000010:2.0:1775767865.010508:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f220. 00000100:00000001:2.0:1775767865.010509:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767865.010511:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099aa680 x1862022294244608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767865.010516:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099aa680 x1862022294244608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767865.010520:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767865.010521:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767865.010522:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767865.010523:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767865.010525:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099aa680 x1862022294244608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767865.010529:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767865.010531:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099aa680 x1862022294244608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767890 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767865.010536:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767865.010538:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767865.010539:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767865.010540:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767865.010541:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767865.010542:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767865.010544:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767865.010546:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767865.010547:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767865.010549:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099aa680 x1862022294244608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767890 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767865.010554:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099aa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294244608:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775767865.010556:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767865.010558:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ab800 x1862022294244736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767865.010563:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767865.010564:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767865.010565:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767865.010566:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767865.010568:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767865.010569:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294244736:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775767865.010572:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767865.010573:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767865.010574:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767865.010575:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775767865.010578:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f800. 02000000:00000001:2.0:1775767865.010579:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767865.010581:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767865.010583:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39c60. 00000400:00000010:2.0:1775767865.010584:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f220. 00000100:00000200:2.0:1775767865.010587:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294244736, portal 10 00000100:00000001:2.0:1775767865.010588:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767865.010589:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640866816 : -112966068684800 : ffff9942099ab800) 00000100:00000040:2.0:1775767865.010593:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ab800 x1862022294244736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767890 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767865.010597:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767865.010598:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767865.010600:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775767865.010601:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294244736, offset 0 00000400:00000200:2.0:1775767865.010604:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767865.010606:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767865.010608:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767865.010610:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767865.010612:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767865.010613:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767865.010615:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767865.010617:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225085:225085:256:4294967295] 192.168.202.156@tcp LPNI seq info [225081:225081:8:4294967295] 00000400:00000200:2.0:1775767865.010620:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767865.010624:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767865.010627:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767865.010629:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3300. 00000800:00000200:2.0:1775767865.010632:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775767865.010633:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775767865.010636:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767865.010638:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000100:2.0:1775767865.010640:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767865.010642:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3300. 00000400:00000200:2.0:1775767865.010644:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775767865.010646:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775767865.010648:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775767865.010649:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775767865.010651:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767865.010653:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ab800 x1862022294244736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767890 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767865.010658:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767865.010659:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ab800 x1862022294244736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767890 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767865.010664:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767865.010665:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775767865.010666:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767865.010668:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:2.0:1775767865.010670:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767865.010671:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299789835 00000100:00000001:2.0:1775767865.010673:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299791165 00000100:00000001:2.0:1775767865.010674:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299789835 00000100:00000001:2.0:1775767865.010676:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299791165 00000100:00000001:2.0:1775767865.010677:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767865.010678:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767865.010679:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767865.010681:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767865/real 1775767865] req@ffff9942099ab800 x1862022294244736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775767865.010686:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767865.010687:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767865.010688:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39c60. 00000400:00000200:2.0:1775767865.010689:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f220 00000400:00000010:2.0:1775767865.010690:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f220. 00000100:00000001:2.0:1775767865.010691:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767865.010693:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ab800 x1862022294244736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767865.010698:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ab800 x1862022294244736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767865.010702:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767865.010703:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767865.010705:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767865.010705:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767865.010708:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ab800 x1862022294244736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767865.010712:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767865.010714:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ab800 x1862022294244736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767890 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767865.010718:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767865.010719:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767865.010720:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767865.010721:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767865.010722:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767865.010724:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767865.010725:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767865.010727:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767865.010728:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767865.010730:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ab800 x1862022294244736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767890 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767865.010735:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294244736:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775767865.010737:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767865.010739:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099aa300 x1862022294244864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767865.010743:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767865.010744:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767865.010746:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767865.010747:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767865.010748:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767865.010750:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099aa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294244864:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767865.010752:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767865.010753:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767865.010754:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767865.010755:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775767865.010757:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1d000. 02000000:00000001:2.0:1775767865.010758:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767865.010760:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767865.010762:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39c60. 00000400:00000010:2.0:1775767865.010764:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f220. 00000100:00000200:2.0:1775767865.010765:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294244864, portal 4 00000100:00000001:2.0:1775767865.010767:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767865.010767:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640861440 : -112966068690176 : ffff9942099aa300) 00000100:00000040:2.0:1775767865.010771:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099aa300 x1862022294244864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767890 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767865.010774:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767865.010776:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767865.010777:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775767865.010779:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294244864, offset 0 00000400:00000200:2.0:1775767865.010781:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767865.010783:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767865.010785:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767865.010787:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767865.010788:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767865.010790:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767865.010791:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767865.010793:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225086:225086:256:4294967295] 192.168.202.156@tcp LPNI seq info [225082:225082:8:4294967295] 00000400:00000200:2.0:1775767865.010796:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767865.010800:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767865.010803:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767865.010805:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3300. 00000800:00000200:2.0:1775767865.010807:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775767865.010809:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775767865.010811:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767865.010813:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000100:2.0:1775767865.010815:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767865.010816:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3300. 00000400:00000200:2.0:1775767865.010818:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775767865.010820:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775767865.010821:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775767865.010822:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775767865.010824:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767865.010826:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099aa300 x1862022294244864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767890 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767865.010830:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767865.010832:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099aa300 x1862022294244864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767890 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767865.010836:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767865.010837:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775767865.010839:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767865.010841:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:2.0:1775767865.010843:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767865.010844:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299789858 00000100:00000001:2.0:1775767865.010845:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299791165 00000100:00000001:2.0:1775767865.010847:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299789858 00000100:00000001:2.0:1775767865.010849:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299791165 00000100:00000001:2.0:1775767865.010850:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767865.010851:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767865.010852:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767865.010854:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767865/real 1775767865] req@ffff9942099aa300 x1862022294244864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775767865.010859:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767865.010860:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767865.010861:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39c60. 00000400:00000200:2.0:1775767865.010862:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f220 00000400:00000010:2.0:1775767865.010863:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f220. 00000100:00000001:2.0:1775767865.010864:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767865.010866:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099aa300 x1862022294244864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767865.010871:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099aa300 x1862022294244864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767865.010874:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767865.010875:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767865.010876:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767865.010877:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767865.010879:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099aa300 x1862022294244864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767865.010884:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767865.010886:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099aa300 x1862022294244864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767890 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767865.010890:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767865.010891:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767865.010892:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767865.010893:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767865.010894:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767865.010896:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767865.010898:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767865.010900:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767865.010901:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767865.010903:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099aa300 x1862022294244864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767890 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767865.010908:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099aa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294244864:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767865.010911:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767865.010912:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767865.010913:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099aa680 x1862022294244608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767890 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767865.010917:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767865.010918:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767865.010919:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1e400. 02000000:00000001:2.0:1775767865.010921:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767865.010922:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767865.010922:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:2.0:1775767865.010924:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767865.010925:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b2800. 02000000:00000001:2.0:1775767865.010926:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767865.010927:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767865.010928:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099aa680. 00000100:00000001:2.0:1775767865.010929:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767865.010930:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767865.010931:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767865.010933:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ab800 x1862022294244736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767890 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767865.010937:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767865.010938:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767865.010939:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f800. 02000000:00000001:2.0:1775767865.010940:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767865.010941:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767865.010942:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:2.0:1775767865.010943:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767865.010943:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b0000. 02000000:00000001:2.0:1775767865.010945:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767865.010946:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767865.010947:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ab800. 00000100:00000001:2.0:1775767865.010948:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767865.010949:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767865.010950:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767865.010952:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099aa300 x1862022294244864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767890 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767865.010956:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767865.010956:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767865.010957:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1d000. 02000000:00000001:2.0:1775767865.010959:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767865.010960:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767865.010961:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:2.0:1775767865.010962:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767865.010962:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b2000. 02000000:00000001:2.0:1775767865.010963:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767865.010964:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767865.010965:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099aa300. 00000100:00000001:2.0:1775767865.010966:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767865.010967:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767865.010968:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767865.010970:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767865.010971:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767865.010972:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775767865.010973:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767865.010974:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767865.010976:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a8a80 x1862022294244992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767865.010980:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767865.010980:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767865.010982:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767865.010983:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767865.010984:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767865.010985:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294244992:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767865.010987:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767865.010989:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767865.010990:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767865.010991:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775767865.010993:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1d000. 02000000:00000001:2.0:1775767865.010994:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767865.010996:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767865.010997:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39c60. 00000400:00000010:2.0:1775767865.010999:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f220. 00000100:00000200:2.0:1775767865.011001:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294244992, portal 4 00000100:00000001:2.0:1775767865.011002:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767865.011003:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640855168 : -112966068696448 : ffff9942099a8a80) 00000100:00000040:2.0:1775767865.011006:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a8a80 x1862022294244992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767890 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767865.011011:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767865.011012:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767865.011014:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775767865.011015:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294244992, offset 0 00000400:00000200:2.0:1775767865.011017:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767865.011019:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767865.011021:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767865.011023:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767865.011024:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767865.011026:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767865.011027:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767865.011029:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225087:225087:256:4294967295] 192.168.202.156@tcp LPNI seq info [225083:225083:8:4294967295] 00000400:00000200:2.0:1775767865.011032:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767865.011036:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767865.011038:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767865.011040:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3300. 00000800:00000200:2.0:1775767865.011043:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775767865.011044:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775767865.011047:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767865.011049:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000100:2.0:1775767865.011051:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767865.011053:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3300. 00000400:00000200:2.0:1775767865.011054:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775767865.011057:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775767865.011059:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775767865.011061:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775767865.011063:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767865.011066:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a8a80 x1862022294244992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767890 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767865.011074:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767865.011077:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a8a80 x1862022294244992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767890 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767865.011083:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767865.011085:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775767865.011088:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767865.011091:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:2.0:1775767865.011094:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767865.011096:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299789861 00000100:00000001:2.0:1775767865.011099:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299791165 00000100:00000001:2.0:1775767865.011101:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299789861 00000100:00000001:2.0:1775767865.011103:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299791165 00000100:00000001:2.0:1775767865.011106:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767865.011108:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767865.011110:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767865.011114:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767865/real 1775767865] req@ffff9942099a8a80 x1862022294244992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775767865.011122:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767865.011123:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767865.011124:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39c60. 00000400:00000200:2.0:1775767865.011126:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f220 00000400:00000010:2.0:1775767865.011127:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f220. 00000100:00000001:2.0:1775767865.011129:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767865.011131:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a8a80 x1862022294244992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767865.011136:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a8a80 x1862022294244992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767865.011140:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767865.011141:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767865.011142:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767865.011143:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767865.011145:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a8a80 x1862022294244992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767865.011150:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767865.011152:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a8a80 x1862022294244992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767890 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767865.011172:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767865.011173:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767865.011175:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767865.011176:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767865.011177:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767865.011179:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767865.011181:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767865.011184:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767865.011185:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767865.011188:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a8a80 x1862022294244992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767890 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767865.011193:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294244992:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767865.011195:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767865.011197:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767865.011211:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a8a80 x1862022294244992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767890 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767865.011216:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767865.011217:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767865.011219:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1d000. 02000000:00000001:2.0:1775767865.011220:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767865.011221:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767865.011222:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:2.0:1775767865.011224:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767865.011225:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b0400. 02000000:00000001:2.0:1775767865.011226:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767865.011227:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767865.011228:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a8a80. 00000100:00000001:2.0:1775767865.011230:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767865.011231:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767865.011232:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767865.011235:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767865.011235:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767865.011237:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775767865.011238:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767865.073340:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767865.073343:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767865.073346:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775767866.033294:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767866.033295:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767866.033298:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767866.033299:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767866.033300:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767866.033301:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767866.033304:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775767866.033304:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767866.033308:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767866.033308:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767866.033311:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767866.033311:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767866.033314:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767866.033316:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767866.033318:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767866.033320:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767866.033322:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767866.033324:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767866.353262:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299791165, at 4299791168 00000100:00000010:1.0:1775767866.353275:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d6660. 00000100:00000001:3.0:1775767867.057237:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767867.057245:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767867.057247:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767867.057251:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:3.0:1775767867.057252:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767867.057256:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767867.057258:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767867.057259:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767867.057263:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767867.057263:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767867.057268:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767867.057269:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767867.057272:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767867.057273:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767867.057278:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767867.057282:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767867.057285:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767867.057289:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767867.121353:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767867.121356:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767867.121359:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775767868.081240:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767868.081240:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767868.081245:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767868.081248:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767868.081249:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767868.081250:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767868.081253:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767868.081253:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767868.081253:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767868.081262:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775767868.081262:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775767868.081262:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767868.081271:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767868.081271:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767868.081271:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767868.081277:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767868.081278:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767868.081278:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767868.081285:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767868.081289:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767868.081292:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767868.081293:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767868.081296:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767868.081300:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767868.081301:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767868.081304:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767868.081307:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767869.105237:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767869.105244:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767869.105246:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767869.105251:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767869.105251:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767869.105254:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767869.105256:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767869.105256:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767869.105258:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767869.105262:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:1.0:1775767869.105263:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775767869.105263:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767869.105269:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767869.105272:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767869.105272:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767869.105274:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767869.105276:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767869.105278:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767869.105280:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767869.105284:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767869.105289:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767869.105289:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767869.105292:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767869.105295:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767869.105298:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767869.105302:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767869.105307:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767869.169344:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767869.169347:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767869.169349:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775767870.129222:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767870.129223:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767870.129229:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767870.129230:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767870.129232:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767870.129233:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767870.129240:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775767870.129240:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767870.129247:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767870.129247:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767870.129253:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767870.129254:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767870.129258:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767870.129261:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767870.129265:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767870.129268:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767870.129272:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767870.129275:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767870.129277:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767870.129280:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767870.129282:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767870.129285:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767870.129290:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767870.129294:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767870.129299:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767870.129303:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767870.129308:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:3.0:1775767870.129317:0:36:0:(rw.c:1481:ll_writepages()) Process entered 00000080:00000001:3.0:1775767870.129322:0:36:0:(file.c:5250:cl_sync_file_range()) Process entered 00000020:00000001:3.0:1775767870.129326:0:36:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:3.0:1775767870.129335:0:36:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff99420693e000. 00000020:00000010:3.0:1775767870.129345:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcfe00. 00000020:00000010:3.0:1775767870.129352:0:36:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210dcec00. 00000020:00000010:3.0:1775767870.129357:0:36:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207981a80. 00000008:00000010:3.0:1775767870.129365:0:36:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994208b50000. 00020000:00000010:3.0:1775767870.129370:0:36:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994218000300. 00008000:00000010:3.0:1775767870.129377:0:36:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994207e08e20. 00000080:00000010:3.0:1775767870.129385:0:36:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99420541b1f8. 00000080:00000010:3.0:1775767870.129390:0:36:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077c0248. 00000020:00000010:3.0:1775767870.129395:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dce800. 00000008:00000010:3.0:1775767870.129399:0:36:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942086f8420. 00020000:00000010:3.0:1775767870.129404:0:36:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994207e8e400. 00008000:00000010:3.0:1775767870.129408:0:36:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994209270a90. 00000080:00000010:3.0:1775767870.129412:0:36:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942090e33c0. 00000020:00000001:3.0:1775767870.129419:0:36:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107590086664 : -112966119464952 : ffff99420693e008) 00000020:00001000:3.0:1775767870.129423:0:36:0:(cl_object.c:831:cl_env_get()) 1@ffff99420693e000 00000020:00000001:3.0:1775767870.129428:0:36:0:(cl_object.c:151:cl_object_top()) ffff99420791a5f8 -> ffff99420791a5f8 00000020:00000001:3.0:1775767870.129432:0:36:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:3.0:1775767870.129435:0:36:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:3.0:1775767870.129438:0:36:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x10176:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:3.0:1775767870.129446:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:3.0:1775767870.129450:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:3.0:1775767870.129452:0:36:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:3.0:1775767870.129455:0:36:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x10176:0x0]io ffff9942077c02a8 type 6 ignore/verify layout 1/0 00020000:00000002:3.0:1775767870.129458:0:36:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207dfb6c8) owner 0000000000000000/ffff994201348000 00020000:00000001:3.0:1775767870.129461:0:36:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:3.0:1775767870.129463:0:36:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:3.0:1775767870.129467:0:36:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00020000:00000001:3.0:1775767870.129470:0:36:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767870.129475:0:36:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:3.0:1775767870.129478:0:36:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00000100:00000001:1.0:1775767870.129480:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00020000:00000001:3.0:1775767870.129482:0:36:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767870.129485:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1775767870.129486:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:3.0:1775767870.129490:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00000001:1.0:1775767870.129491:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00020000:00000001:3.0:1775767870.129493:0:36:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00020000:00000002:3.0:1775767870.129495:0:36:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207dfb6c8) owner 0000000000000000/ffff994201348000 00000020:00000001:3.0:1775767870.129501:0:36:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767870.129502:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fc380 x1862022294245120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775767870.129504:0:36:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:3.0:1775767870.129507:0:36:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00020000:00000001:3.0:1775767870.129511:0:36:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00020000:00200000:3.0:1775767870.129516:0:36:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00000002:3.0:1775767870.129521:0:36:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:3.0:1775767870.129529:0:36:0:(lov_io.c:155:lov_sub_get()) Process entered 00020000:00000001:3.0:1775767870.129533:0:36:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000100:00000001:1.0:1775767870.129549:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767870.129552:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767870.129558:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775767870.129561:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767870.129568:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767870.129582:0:36:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00100000:1.0:1775767870.129584:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294245120:192.168.202.156@tcp:250:kworker.0 00000020:00000010:3.0:1775767870.129586:0:36:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff99420693ecf0. 00000020:00000010:3.0:1775767870.129592:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcf600. 00000100:00000001:1.0:1775767870.129597:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000010:3.0:1775767870.129597:0:36:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210dcf200. 00000020:00000010:3.0:1775767870.129602:0:36:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207981d80. 02000000:00000001:1.0:1775767870.129603:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775767870.129607:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:3.0:1775767870.129609:0:36:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994208b51650. 02000000:00000001:1.0:1775767870.129612:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00020000:00000010:3.0:1775767870.129613:0:36:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994218000000. 02000000:00000010:1.0:1775767870.129619:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543f000. 00008000:00000010:3.0:1775767870.129621:0:36:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994207e0ade8. 02000000:00000001:1.0:1775767870.129623:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:3.0:1775767870.129629:0:36:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99420541b0a8. 00000400:00000200:1.0:1775767870.129633:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000080:00000010:3.0:1775767870.129637:0:36:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077c1da8. 00000400:00000010:1.0:1775767870.129638:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023cf78. 00000400:00000010:1.0:1775767870.129647:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf63b8. 00000100:00000200:1.0:1775767870.129654:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294245120, portal 25 00000100:00000001:1.0:1775767870.129658:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775767870.129660:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641197440 : -112966068354176 : ffff9942099fc380) 00000100:00000040:1.0:1775767870.129668:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fc380 x1862022294245120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767895 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767870.129685:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775767870.129690:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775767870.129695:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6dd0. 00000100:00000200:1.0:1775767870.129700:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294245120, offset 0 00000400:00000200:1.0:1775767870.129710:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775767870.129720:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767870.129727:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775767870.129734:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000010:3.0:1775767870.129741:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dce400. 00000400:00000200:1.0:1775767870.129742:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000008:00000010:3.0:1775767870.129748:0:36:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942086f8160. 00000400:00000200:1.0:1775767870.129750:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00020000:00000010:3.0:1775767870.129752:0:36:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994207e8ed00. 00008000:00000010:3.0:1775767870.129757:0:36:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994209270ef0. 00000400:00000200:1.0:1775767870.129758:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000080:00000010:3.0:1775767870.129762:0:36:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942090e3960. 00000400:00000200:1.0:1775767870.129768:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225088:225088:256:4294967295] 192.168.202.156@tcp LPNI seq info [225084:225084:8:4294967295] 00000020:00000001:3.0:1775767870.129774:0:36:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107590089976 : -112966119461640 : ffff99420693ecf8) 00000020:00001000:3.0:1775767870.129778:0:36:0:(cl_object.c:831:cl_env_get()) 1@ffff99420693ecf0 00000400:00000200:1.0:1775767870.129783:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:3.0:1775767870.129783:0:36:0:(cl_object.c:151:cl_object_top()) ffff99420791ae90 -> ffff99420791a5f8 00000400:00000200:1.0:1775767870.129800:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775767870.129807:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000020:00000001:3.0:1775767870.129822:0:36:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:3.0:1775767870.129826:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000800:00000010:1.0:1775767870.129829:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541800. 00000020:00000001:3.0:1775767870.129829:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:3.0:1775767870.129833:0:36:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767870.129837:0:36:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767870.129842:0:36:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107612435560 : -112966097116056 : ffff994207e8e468) 00000800:00000200:1.0:1775767870.129843:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000001:3.0:1775767870.129848:0:36:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000020:00000001:3.0:1775767870.129851:0:36:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775767870.129852:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00020000:00000001:3.0:1775767870.129855:0:36:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767870.129860:0:36:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767870.129864:0:36:0:(cl_io.c:314:cl_io_lock()) Process entered 00000800:00000200:1.1:1775767870.129864:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00020000:00000001:3.0:1775767870.129866:0:36:0:(lov_io.c:1108:lov_io_lock()) Process entered 00020000:00000001:3.0:1775767870.129870:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775767870.129872:0:36:0:(cl_io.c:314:cl_io_lock()) Process entered 00000800:00000200:1.1:1775767870.129872:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000020:00000001:3.0:1775767870.129874:0:36:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:3.0:1775767870.129877:0:36:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767870.129880:0:36:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1775767870.129886:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00020000:00000001:3.0:1775767870.129886:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1775767870.129891:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541800. 00020000:00000001:3.0:1775767870.129891:0:36:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767870.129896:0:36:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:3.0:1775767870.129898:0:36:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767870.129899:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000001:3.0:1775767870.129902:0:36:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767870.129907:0:36:0:(cl_io.c:528:cl_io_start()) Process entered 00000400:00000200:1.0:1775767870.129908:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00020000:00000001:3.0:1775767870.129910:0:36:0:(lov_io.c:1114:lov_io_start()) Process entered 00020000:00000001:3.0:1775767870.129913:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000400:00000200:1.0:1775767870.129914:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6dd0 00000020:00000001:3.0:1775767870.129916:0:36:0:(cl_io.c:528:cl_io_start()) Process entered 00000400:00000010:1.0:1775767870.129919:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6dd0. 00000008:00000001:3.0:1775767870.129919:0:36:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000008:00000001:3.0:1775767870.129924:0:36:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000100:00000001:1.0:1775767870.129925:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000008:00000001:3.0:1775767870.129931:0:36:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000100:00000200:1.0:1775767870.129935:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fc380 x1862022294245120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767895 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000020:3.0:1775767870.129935:0:36:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:3.0:1775767870.129938:0:36:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1775767870.129944:0:36:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000008:00000001:3.0:1775767870.129947:0:36:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000008:00000001:3.0:1775767870.129950:0:36:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107754940544 : -112965954611072 : ffff994210675880) 00000100:00000001:1.0:1775767870.129954:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000008:00000020:3.0:1775767870.129954:0:36:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994210675880 ready 0|-|- wr 1|-|+ rd 0|- 10 in flight 00000100:00000040:1.0:1775767870.129960:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fc380 x1862022294245120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767895 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1775767870.129962:0:36:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:3.0:1775767870.129965:0:36:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:3.0:1775767870.129968:0:36:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1775767870.129975:0:36:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000100:00000001:1.0:1775767870.129980:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000008:00000020:3.0:1775767870.129980:0:36:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994218232fc0 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000100:00000001:1.0:1775767870.129985:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000008:00000001:3.0:1775767870.129990:0:36:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767870.129994:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1775767870.129996:0:36:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767870.130001:0:36:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767870.130005:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:3.0:1775767870.130006:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767870.130009:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299796285 00020000:00000001:3.0:1775767870.130011:0:36:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775767870.130016:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cbaae0. 00000020:00000001:3.0:1775767870.130017:0:36:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767870.130023:0:36:0:(cl_io.c:555:cl_io_end()) Process entered 00000100:00000001:1.0:1775767870.130025:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00020000:00000001:3.0:1775767870.130026:0:36:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00020000:00000001:3.0:1775767870.130030:0:36:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000100:00000001:1.0:1775767870.130033:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299796285 00000020:00000001:3.0:1775767870.130033:0:36:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000001:3.0:1775767870.130037:0:36:0:(cl_io.c:563:cl_io_end()) Process leaving 00000100:00000010:1.0:1775767870.130038:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba180. 00020000:00000001:3.0:1775767870.130040:0:36:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767870.130043:0:36:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000020:00000001:3.0:1775767870.130046:0:36:0:(cl_io.c:563:cl_io_end()) Process leaving 00000020:00000001:3.0:1775767870.130064:0:36:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000100:00000001:1.0:1775767870.130068:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00020000:00000001:3.0:1775767870.130068:0:36:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00020000:00000001:3.0:1775767870.130072:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775767870.130075:0:36:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000100:00000001:1.0:1775767870.130077:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1775767870.130079:0:36:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000100:00000001:1.0:1775767870.130083:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767870.130083:0:36:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767870.130089:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767870.130090:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00020000:00000001:3.0:1775767870.130094:0:36:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000100:00100000:1.0:1775767870.130098:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767870/real 1775767870] req@ffff9942099fc380 x1862022294245120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775767870.130098:0:36:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:3.0:1775767870.130103:0:36:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000020:00000001:3.0:1775767870.130106:0:36:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000001:3.0:1775767870.130110:0:36:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:3.0:1775767870.130129:0:36:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00000400:00000001:1.0:1775767870.130132:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00020000:00000001:3.0:1775767870.130133:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000400:00000001:1.0:1775767870.130134:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:3.0:1775767870.130136:0:36:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000400:00000010:1.0:1775767870.130137:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023cf78. 00000020:00000001:3.0:1775767870.130140:0:36:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000400:00000200:1.0:1775767870.130143:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf63b8 00020000:00000001:3.0:1775767870.130143:0:36:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775767870.130147:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf63b8. 00020000:00000001:3.0:1775767870.130148:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775767870.130182:0:36:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000020:00000001:3.0:1775767870.130184:0:36:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000100:00000001:1.0:1775767870.130185:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:3.0:1775767870.130186:0:36:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767870.130190:0:36:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:3.0:1775767870.130192:0:36:0:(lov_io.c:675:lov_io_fini()) Process entered 00000100:00000200:1.0:1775767870.130194:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fc380 x1862022294245120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:3.0:1775767870.130195:0:36:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:3.0:1775767870.130198:0:36:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:3.0:1775767870.130201:0:36:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:3.0:1775767870.130204:0:36:0:(cl_object.c:925:cl_env_put()) 1@ffff99420693ecf0 00000020:00000010:3.0:1775767870.130210:0:36:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210dcf200. 00000020:00000010:3.0:1775767870.130216:0:36:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207981d80. 00000008:00000010:3.0:1775767870.130222:0:36:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994208b51650. 00000100:00000200:1.0:1775767870.130224:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fc380 x1862022294245120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000010:3.0:1775767870.130228:0:36:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994218000000. 00008000:00000010:3.0:1775767870.130234:0:36:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994207e0ade8. 00000080:00000010:3.0:1775767870.130240:0:36:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99420541b0a8. 00000100:00000001:1.0:1775767870.130242:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775767870.130246:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000080:00000010:3.0:1775767870.130246:0:36:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077c1da8. 00000100:00000001:1.0:1775767870.130250:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000010:3.0:1775767870.130252:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcf600. 00000100:00000001:1.0:1775767870.130253:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00000010:3.0:1775767870.130258:0:36:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942086f8160. 00000100:00100000:1.0:1775767870.130261:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fc380 x1862022294245120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000010:3.0:1775767870.130263:0:36:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994207e8ed00. 00008000:00000010:3.0:1775767870.130269:0:36:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994209270ef0. 00000080:00000010:3.0:1775767870.130274:0:36:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942090e3960. 00000020:00000010:3.0:1775767870.130280:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dce400. 00000020:00000010:3.0:1775767870.130285:0:36:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff99420693ecf0. 00020000:00000001:3.0:1775767870.130290:0:36:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00000100:00000001:1.0:1775767870.130295:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00020000:00000001:3.0:1775767870.130295:0:36:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:3.0:1775767870.130300:0:36:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000100:00000040:1.0:1775767870.130303:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fc380 x1862022294245120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767895 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00200000:3.0:1775767870.130305:0:36:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x10176:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:3.0:1775767870.130317:0:36:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:3.0:1775767870.130320:0:36:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:3.0:1775767870.130325:0:36:0:(cl_object.c:925:cl_env_put()) 1@ffff99420693e000 00000100:00000001:1.0:1775767870.130327:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000010:3.0:1775767870.130331:0:36:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210dcec00. 00000100:00000001:1.0:1775767870.130333:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767870.130338:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767870.130339:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:3.0:1775767870.130339:0:36:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207981a80. 00000100:00000001:1.0:1775767870.130344:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000008:00000010:3.0:1775767870.130345:0:36:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994208b50000. 00000100:00000001:1.0:1775767870.130348:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00020000:00000010:3.0:1775767870.130351:0:36:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994218000300. 00000100:00080000:1.0:1775767870.130352:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00008000:00000010:3.0:1775767870.130356:0:36:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994207e08e20. 00000080:00000010:3.0:1775767870.130361:0:36:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99420541b1f8. 00000100:00080000:1.0:1775767870.130362:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000080:00000010:3.0:1775767870.130366:0:36:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077c0248. 00000100:00000001:1.0:1775767870.130367:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000010:3.0:1775767870.130371:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcfe00. 00000100:00000040:1.0:1775767870.130376:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fc380 x1862022294245120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767895 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000010:3.0:1775767870.130376:0:36:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942086f8420. 00020000:00000010:3.0:1775767870.130381:0:36:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994207e8e400. 00008000:00000010:3.0:1775767870.130387:0:36:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994209270a90. 00000080:00000010:3.0:1775767870.130392:0:36:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942090e33c0. 00000020:00000010:3.0:1775767870.130397:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dce800. 00000020:00000010:3.0:1775767870.130404:0:36:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff99420693e000. 00000100:00100000:1.0:1775767870.130406:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294245120:192.168.202.156@tcp:250:kworker.0 00000080:00000001:3.0:1775767870.130410:0:36:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767870.130413:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767870.130417:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000080:00000001:3.0:1775767870.130417:0:36:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767870.130423:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fc380 x1862022294245120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767895 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767870.130437:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775767870.130441:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775767870.130445:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543f000. 02000000:00000001:1.0:1775767870.130450:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775767870.130452:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775767870.130456:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:1.0:1775767870.130461:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775767870.130464:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002a800. 02000000:00000001:1.0:1775767870.130468:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775767870.130470:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775767870.130473:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fc380. 00000100:00000001:1.0:1775767870.130478:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775767870.130480:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767870.130485:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767870.130491:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767870.130493:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767870.130497:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775767870.130500:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767870.130502:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775767870.130508:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fed80 x1862022294245248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767870.130520:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767870.130522:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767870.130525:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775767870.130528:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767870.130534:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767870.130542:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294245248:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775767870.130549:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775767870.130552:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775767870.130555:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767870.130560:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775767870.130565:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543f000. 02000000:00000001:1.0:1775767870.130571:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767870.130576:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775767870.130578:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023cf78. 00000400:00000010:1.0:1775767870.130581:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf63b8. 00000100:00000200:1.0:1775767870.130584:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294245248, portal 10 00000100:00000001:1.0:1775767870.130586:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775767870.130587:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641208192 : -112966068343424 : ffff9942099fed80) 00000100:00000040:1.0:1775767870.130593:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fed80 x1862022294245248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767895 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767870.130598:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775767870.130599:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775767870.130601:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6dd0. 00000100:00000200:1.0:1775767870.130602:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294245248, offset 0 00000400:00000200:1.0:1775767870.130605:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775767870.130608:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767870.130610:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775767870.130613:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775767870.130614:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775767870.130616:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775767870.130618:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775767870.130620:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225089:225089:256:4294967295] 192.168.202.156@tcp LPNI seq info [225085:225085:8:4294967295] 00000400:00000200:1.0:1775767870.130623:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767870.130627:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775767870.130630:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767870.130633:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541800. 00000800:00000200:1.0:1775767870.130635:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775767870.130637:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775767870.130641:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767870.130642:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1775767870.130645:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767870.130646:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541800. 00000400:00000200:1.0:1775767870.130648:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775767870.130651:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775767870.130653:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6dd0 00000400:00000010:1.0:1775767870.130654:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6dd0. 00000100:00000001:1.0:1775767870.130655:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775767870.130657:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fed80 x1862022294245248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767895 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767870.130663:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767870.130665:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fed80 x1862022294245248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767895 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767870.130670:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767870.130671:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775767870.130673:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767870.130675:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:1.0:1775767870.130678:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767870.130679:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299794963 00000100:00000001:1.0:1775767870.130681:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299796285 00000100:00000001:1.0:1775767870.130682:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299794963 00000100:00000001:1.0:1775767870.130684:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299796285 00000100:00000001:1.0:1775767870.130685:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775767870.130686:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767870.130688:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775767870.130690:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767870/real 1775767870] req@ffff9942099fed80 x1862022294245248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775767870.130699:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775767870.130700:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775767870.130701:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023cf78. 00000400:00000200:1.0:1775767870.130703:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf63b8 00000400:00000010:1.0:1775767870.130703:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf63b8. 00000100:00000001:1.0:1775767870.130705:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775767870.130706:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fed80 x1862022294245248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775767870.130711:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fed80 x1862022294245248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767870.130715:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775767870.130716:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767870.130718:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767870.130719:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767870.130721:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fed80 x1862022294245248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767870.130725:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775767870.130727:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fed80 x1862022294245248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767895 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767870.130731:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775767870.130733:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767870.130734:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767870.130734:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767870.130736:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775767870.130737:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775767870.130739:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767870.130741:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775767870.130742:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775767870.130745:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fed80 x1862022294245248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767895 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767870.130750:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294245248:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775767870.130753:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775767870.130755:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fd180 x1862022294245376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767870.130759:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767870.130760:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767870.130761:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775767870.130762:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767870.130763:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767870.130765:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294245376:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775767870.130766:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775767870.130767:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775767870.130768:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767870.130769:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775767870.130771:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543d800. 02000000:00000001:1.0:1775767870.130773:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767870.130774:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775767870.130776:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023cf78. 00000400:00000010:1.0:1775767870.130777:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf63b8. 00000100:00000200:1.0:1775767870.130780:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294245376, portal 10 00000100:00000001:1.0:1775767870.130781:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775767870.130782:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641201024 : -112966068350592 : ffff9942099fd180) 00000100:00000040:1.0:1775767870.130785:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fd180 x1862022294245376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767895 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767870.130806:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775767870.130808:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775767870.130809:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6dd0. 00000100:00000200:1.0:1775767870.130811:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294245376, offset 0 00000400:00000200:1.0:1775767870.130813:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775767870.130816:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767870.130818:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775767870.130820:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775767870.130822:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775767870.130824:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775767870.130825:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775767870.130827:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225090:225090:256:4294967295] 192.168.202.156@tcp LPNI seq info [225086:225086:8:4294967295] 00000400:00000200:1.0:1775767870.130830:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767870.130834:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775767870.130837:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767870.130839:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541800. 00000800:00000200:1.0:1775767870.130841:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775767870.130843:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775767870.130846:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767870.130848:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1775767870.130850:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767870.130852:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541800. 00000400:00000200:1.0:1775767870.130873:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775767870.130875:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775767870.130877:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6dd0 00000400:00000010:1.0:1775767870.130878:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6dd0. 00000100:00000001:1.0:1775767870.130880:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775767870.130882:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fd180 x1862022294245376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767895 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767870.130887:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767870.130888:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fd180 x1862022294245376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767895 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767870.130892:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767870.130893:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775767870.130895:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767870.130897:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:1.0:1775767870.130899:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767870.130901:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299794955 00000100:00000001:1.0:1775767870.130902:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299796285 00000100:00000001:1.0:1775767870.130904:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299794955 00000100:00000001:1.0:1775767870.130905:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299796285 00000100:00000001:1.0:1775767870.130906:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775767870.130908:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767870.130909:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775767870.130911:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767870/real 1775767870] req@ffff9942099fd180 x1862022294245376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775767870.130915:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775767870.130916:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775767870.130917:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023cf78. 00000400:00000200:1.0:1775767870.130918:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf63b8 00000400:00000010:1.0:1775767870.130919:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf63b8. 00000100:00000001:1.0:1775767870.130921:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775767870.130923:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fd180 x1862022294245376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775767870.130928:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fd180 x1862022294245376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767870.130932:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775767870.130932:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767870.130933:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767870.130934:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767870.130936:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fd180 x1862022294245376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767870.130940:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775767870.130942:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fd180 x1862022294245376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767895 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767870.130946:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775767870.130948:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767870.130949:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767870.130950:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767870.130951:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775767870.130953:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775767870.130955:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767870.130957:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775767870.130958:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775767870.130960:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fd180 x1862022294245376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767895 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767870.130965:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294245376:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775767870.130967:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775767870.130969:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ff480 x1862022294245504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767870.130973:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767870.130974:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767870.130975:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775767870.130977:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767870.130978:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767870.130979:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294245504:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775767870.130981:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775767870.130982:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775767870.130983:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767870.130984:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775767870.130986:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543dc00. 02000000:00000001:1.0:1775767870.130987:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767870.130989:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775767870.130991:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023cf78. 00000400:00000010:1.0:1775767870.130993:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf63b8. 00000100:00000200:1.0:1775767870.130995:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294245504, portal 4 00000100:00000001:1.0:1775767870.130996:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775767870.130997:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641209984 : -112966068341632 : ffff9942099ff480) 00000100:00000040:1.0:1775767870.131001:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ff480 x1862022294245504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767895 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767870.131005:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775767870.131006:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775767870.131008:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6dd0. 00000100:00000200:1.0:1775767870.131009:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294245504, offset 0 00000400:00000200:1.0:1775767870.131011:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775767870.131013:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767870.131015:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775767870.131017:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775767870.131018:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775767870.131020:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775767870.131021:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775767870.131023:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225091:225091:256:4294967295] 192.168.202.156@tcp LPNI seq info [225087:225087:8:4294967295] 00000400:00000200:1.0:1775767870.131026:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767870.131030:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775767870.131032:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767870.131034:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541800. 00000800:00000200:1.0:1775767870.131036:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775767870.131038:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775767870.131041:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767870.131042:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1775767870.131045:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767870.131046:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541800. 00000400:00000200:1.0:1775767870.131048:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775767870.131050:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775767870.131051:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6dd0 00000400:00000010:1.0:1775767870.131052:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6dd0. 00000100:00000001:1.0:1775767870.131054:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775767870.131055:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ff480 x1862022294245504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767895 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767870.131065:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767870.131067:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ff480 x1862022294245504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767895 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767870.131071:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767870.131072:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775767870.131073:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767870.131075:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:1.0:1775767870.131077:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767870.131078:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299794978 00000100:00000001:1.0:1775767870.131079:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299796285 00000100:00000001:1.0:1775767870.131080:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299794978 00000100:00000001:1.0:1775767870.131082:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299796285 00000100:00000001:1.0:1775767870.131083:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775767870.131084:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767870.131086:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775767870.131087:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767870/real 1775767870] req@ffff9942099ff480 x1862022294245504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775767870.131092:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775767870.131093:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775767870.131093:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023cf78. 00000400:00000200:1.0:1775767870.131095:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf63b8 00000400:00000010:1.0:1775767870.131096:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf63b8. 00000100:00000001:1.0:1775767870.131097:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775767870.131098:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ff480 x1862022294245504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775767870.131104:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ff480 x1862022294245504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767870.131108:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775767870.131109:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767870.131110:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767870.131111:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767870.131113:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ff480 x1862022294245504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767870.131117:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775767870.131120:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ff480 x1862022294245504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767895 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767870.131123:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775767870.131125:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767870.131126:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767870.131127:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767870.131128:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775767870.131130:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775767870.131131:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767870.131133:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775767870.131134:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775767870.131136:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ff480 x1862022294245504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767895 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767870.131141:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294245504:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775767870.131143:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767870.131144:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767870.131146:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fed80 x1862022294245248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767895 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767870.131191:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775767870.131193:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775767870.131195:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543f000. 02000000:00000001:1.0:1775767870.131199:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775767870.131201:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775767870.131203:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:1.0:1775767870.131205:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775767870.131206:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028000. 02000000:00000001:1.0:1775767870.131209:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775767870.131210:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775767870.131211:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fed80. 00000100:00000001:1.0:1775767870.131213:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775767870.131214:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767870.131216:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767870.131219:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fd180 x1862022294245376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767895 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767870.131223:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775767870.131224:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775767870.131225:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543d800. 02000000:00000001:1.0:1775767870.131226:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775767870.131227:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775767870.131228:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:1.0:1775767870.131229:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775767870.131230:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002b800. 02000000:00000001:1.0:1775767870.131231:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775767870.131231:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775767870.131232:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fd180. 00000100:00000001:1.0:1775767870.131233:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775767870.131234:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767870.131235:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767870.131237:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ff480 x1862022294245504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767895 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767870.131240:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775767870.131241:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775767870.131242:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543dc00. 02000000:00000001:1.0:1775767870.131243:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775767870.131244:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775767870.131244:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:1.0:1775767870.131245:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775767870.131246:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029400. 02000000:00000001:1.0:1775767870.131247:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775767870.131249:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775767870.131249:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ff480. 00000100:00000001:1.0:1775767870.131251:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775767870.131252:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767870.131253:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767870.131255:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767870.131256:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767870.131258:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775767870.131259:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767870.131260:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775767870.131262:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fd880 x1862022294245632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767870.131267:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767870.131268:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767870.131269:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775767870.131270:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767870.131272:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767870.131273:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294245632:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775767870.131275:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775767870.131276:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775767870.131277:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767870.131278:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775767870.131281:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543dc00. 02000000:00000001:1.0:1775767870.131282:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767870.131284:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775767870.131286:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023cf78. 00000400:00000010:1.0:1775767870.131288:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf63b8. 00000100:00000200:1.0:1775767870.131290:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294245632, portal 4 00000100:00000001:1.0:1775767870.131291:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775767870.131292:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641202816 : -112966068348800 : ffff9942099fd880) 00000100:00000040:1.0:1775767870.131296:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fd880 x1862022294245632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767895 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767870.131300:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775767870.131302:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775767870.131303:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6dd0. 00000100:00000200:1.0:1775767870.131305:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294245632, offset 0 00000400:00000200:1.0:1775767870.131307:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775767870.131310:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767870.131312:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775767870.131314:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775767870.131316:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775767870.131318:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775767870.131319:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775767870.131321:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225092:225092:256:4294967295] 192.168.202.156@tcp LPNI seq info [225088:225088:8:4294967295] 00000400:00000200:1.0:1775767870.131324:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767870.131328:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775767870.131331:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767870.131332:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541800. 00000800:00000200:1.0:1775767870.131335:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775767870.131337:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775767870.131339:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767870.131341:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1775767870.131344:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767870.131345:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541800. 00000400:00000200:1.0:1775767870.131347:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775767870.131349:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775767870.131351:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6dd0 00000400:00000010:1.0:1775767870.131352:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6dd0. 00000100:00000001:1.0:1775767870.131354:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775767870.131356:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fd880 x1862022294245632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767895 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767870.131361:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767870.131363:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fd880 x1862022294245632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767895 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767870.131367:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767870.131368:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775767870.131399:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767870.131403:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:1.0:1775767870.131407:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767870.131409:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299794982 00000100:00000001:1.0:1775767870.131411:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299796285 00000100:00000001:1.0:1775767870.131413:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299794982 00000100:00000001:1.0:1775767870.131416:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299796285 00000100:00000001:1.0:1775767870.131418:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775767870.131420:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767870.131422:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775767870.131426:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767870/real 1775767870] req@ffff9942099fd880 x1862022294245632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775767870.131434:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775767870.131435:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775767870.131436:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023cf78. 00000400:00000200:1.0:1775767870.131438:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf63b8 00000400:00000010:1.0:1775767870.131439:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf63b8. 00000100:00000001:1.0:1775767870.131441:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775767870.131443:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fd880 x1862022294245632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775767870.131448:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fd880 x1862022294245632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767870.131452:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775767870.131453:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767870.131454:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767870.131455:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767870.131457:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fd880 x1862022294245632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767870.131462:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775767870.131464:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fd880 x1862022294245632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767895 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767870.131469:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775767870.131470:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767870.131471:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767870.131472:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767870.131474:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775767870.131476:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775767870.131477:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767870.131479:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775767870.131481:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775767870.131483:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fd880 x1862022294245632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767895 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767870.131491:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294245632:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775767870.131495:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767870.131497:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767870.131500:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fd880 x1862022294245632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767895 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767870.131506:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775767870.131508:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775767870.131510:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543dc00. 02000000:00000001:1.0:1775767870.131512:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775767870.131513:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775767870.131515:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:1.0:1775767870.131517:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775767870.131518:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028400. 02000000:00000001:1.0:1775767870.131520:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775767870.131521:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775767870.131523:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fd880. 00000100:00000001:1.0:1775767870.131525:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775767870.131526:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767870.131528:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767870.131531:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767870.131532:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767870.131534:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775767870.131535:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767871.153202:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767871.153208:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767871.153210:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767871.153215:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767871.153218:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767871.153222:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767871.153224:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767871.153226:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767871.153229:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767871.153241:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767871.153246:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767871.153248:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767871.153252:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767871.153254:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767871.153255:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767871.153259:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767871.153260:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767871.153263:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767871.153264:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767871.153268:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767871.153270:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775767871.153271:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767871.153276:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767871.153280:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767871.153283:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767871.153285:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767871.153288:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767871.217407:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767871.217410:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767871.217412:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775767871.473222:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299796285, at 4299796288 00000100:00000010:1.0:1775767871.473234:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cba180. 00000100:00000001:1.0:1775767872.177268:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767872.177268:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767872.177277:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767872.177277:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767872.177282:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767872.177282:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767872.177292:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775767872.177292:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767872.177299:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767872.177299:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767872.177305:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767872.177306:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767872.177312:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767872.177316:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767872.177321:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767872.177321:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767872.177324:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767872.177327:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767872.177363:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767872.177369:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767872.177371:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767872.177375:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775767872.177378:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767872.177381:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767872.177383:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767872.177385:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767872.177387:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767873.201293:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767873.201293:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767873.201297:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767873.201305:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767873.201306:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767873.201306:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767873.201311:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767873.201312:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767873.201312:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767873.201327:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:1.0:1775767873.201328:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775767873.201328:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767873.201338:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767873.201339:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767873.201340:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767873.201343:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767873.201344:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767873.201345:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767873.201358:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767873.201361:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767873.201365:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767873.201366:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767873.201369:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767873.201373:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767873.201373:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767873.201376:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767873.201380:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767873.265412:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767873.265417:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767873.265423:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775767874.225256:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767874.225259:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767874.225264:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767874.225264:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767874.225267:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767874.225269:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767874.225273:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775767874.225274:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767874.225276:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767874.225281:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767874.225282:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767874.225283:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767874.225287:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767874.225287:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767874.225288:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767874.225301:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00100000:2.0:1775767874.225302:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767874.225303:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767874.225309:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767874.225310:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767874.225310:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767874.225315:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767874.225318:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767874.225324:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767874.225329:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767874.225332:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767874.225336:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767875.249244:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767875.249251:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767875.249253:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767875.249259:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767875.249263:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767875.249265:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767875.249266:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767875.249272:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767875.249272:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767875.249272:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767875.249274:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767875.249279:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767875.249285:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00100000:3.0:1775767875.249285:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:1.0:1775767875.249286:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775767875.249291:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767875.249294:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767875.249294:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767875.249296:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767875.249300:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767875.249301:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767875.249306:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767875.249309:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767875.249315:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767875.249315:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767875.249318:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767875.249321:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775767875.249341:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767875.249352:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775767875.249356:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775767875.249363:0:36:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775767875.249372:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775767875.249379:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5127 00000400:00000200:2.0:1775767875.249394:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767875.249404:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775767875.249412:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775767875.249422:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767875.249433:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767875.249439:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767875.249451:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775767875.249457:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775767875.249460:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:2.0:1775767875.249466:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775767875.249470:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:2.0:1775767875.249476:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:2.0:1775767875.249481:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:2.0:1775767875.249493:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767875.249497:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767875.249509:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767875.249512:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775767875.249517:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:2.0:1775767875.249527:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775767875.249538:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767875.249547:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775767875.249556:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767875.249564:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767875.249569:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775767875.249576:0:36:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775767875.249590:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dff100. 00000020:00000040:2.0:1775767875.249600:0:36:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:2.0:1775767875.249613:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775767875.249618:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767875.249627:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775767875.249635:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775767875.249641:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775767875.249651:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1d000. 00000100:00000001:2.0:1775767875.249661:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767875.249673:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dff100 x1862022294245760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775767875.249699:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775767875.249707:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767875.249716:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3e64ae1] to pc [ptlrpcd_rcv+-1] req@ffff994207dff100 x1862022294245760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767875.249752:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775767875.249761:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767875.249764:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775767875.249766:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775767875.249770:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775767875.249776:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775767875.249781:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:2.0:1775767875.249782:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5127 00000100:00000001:1.0:1775767875.249790:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000400:00000200:2.0:1775767875.249794:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767875.249798:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000400:00000200:2.0:1775767875.249801:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775767875.249807:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775767875.249815:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767875.249823:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767875.249826:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dff100 x1862022294245760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775767875.249831:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767875.249839:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775767875.249849:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775767875.249855:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:2.0:1775767875.249862:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775767875.249867:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000100:00000001:1.0:1775767875.249869:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767875.249875:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767875.249875:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:2.0:1775767875.249886:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 02000000:00000001:1.0:1775767875.249888:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775767875.249893:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1775767875.249900:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 02000000:00000001:1.0:1775767875.249901:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767875.249911:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:1.0:1775767875.249916:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294245760:192.168.202.156@tcp:250:kworker.0 00000100:00000040:2.0:1775767875.249923:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767875.249926:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1775767875.249935:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 02000000:00000001:1.0:1775767875.249936:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:2.0:1775767875.249944:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 02000000:00000001:1.0:1775767875.249945:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775767875.249959:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 02000000:00000001:1.0:1775767875.249961:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1775767875.249969:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775767875.249970:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543dc00. 02000000:00000001:2.0:1775767875.249973:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775767875.249974:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767875.249985:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:2.0:1775767875.249985:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:1.0:1775767875.249994:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023cf78. 02000000:00000001:2.0:1775767875.249999:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775767875.250003:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf63b8. 00000100:00000001:2.0:1775767875.250007:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000200:1.0:1775767875.250014:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294245760, portal 25 00000008:00100000:2.0:1775767875.250014:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775767875.250026:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767875.250027:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767875.250032:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767875.250034:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611848960 : -112966097702656 : ffff994207dff100) 00000100:00000010:2.0:1775767875.250038:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfdc00. 00000020:00000040:2.0:1775767875.250045:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000100:00000040:1.0:1775767875.250047:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dff100 x1862022294245760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767900 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1775767875.250050:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775767875.250054:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767875.250063:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775767875.250068:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775767875.250076:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767875.250079:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000010:2.0:1775767875.250088:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1f800. 00000100:00000040:1.0:1775767875.250089:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775767875.250114:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6dd0. 00000100:00000200:1.0:1775767875.250120:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294245760, offset 0 00000400:00000200:1.0:1775767875.250128:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00000001:2.0:1775767875.250129:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767875.250141:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00100000:2.0:1775767875.250142:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfdc00 x1862022294245888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775767875.250172:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775767875.250177:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775767875.250182:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775767875.250186:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775767875.250200:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00000001:2.0:1775767875.250211:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1775767875.250212:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225093:225093:256:4294967295] 192.168.202.156@tcp LPNI seq info [225089:225089:8:4294967295] 00000100:00000001:2.0:1775767875.250219:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767875.250228:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000040:2.0:1775767875.250232:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003e25603c] to pc [ptlrpcd_rcv+-1] req@ffff994207dfdc00 x1862022294245888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775767875.250257:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775767875.250291:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767875.250307:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541800. 00000100:00000001:2.0:1775767875.250308:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775767875.250315:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:1.0:1775767875.250325:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00080000:2.0:1775767875.250332:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000200:1.0:1775767875.250336:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000100:00000001:2.0:1775767875.250337:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775767875.250343:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775767875.250357:0:36:0:(import.c:484:import_select_connection()) Process entered 00000800:00000200:1.1:1775767875.250357:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00080000:2.0:1775767875.250368:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5127 00000800:00000200:1.1:1775767875.250370:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000400:00000200:2.0:1775767875.250380:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000800:00000100:1.0:1775767875.250381:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767875.250390:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541800. 00000400:00000200:1.0:1775767875.250395:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775767875.250395:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775767875.250404:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775767875.250411:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000200:1.0:1775767875.250417:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000100:00000001:2.0:1775767875.250421:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1775767875.250426:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6dd0 00000100:00000040:2.0:1775767875.250429:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000010:1.0:1775767875.250431:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6dd0. 00000100:00000001:2.0:1775767875.250440:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775767875.250444:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:1.0:1775767875.250445:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000020:00000040:2.0:1775767875.250448:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:2.0:1775767875.250451:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775767875.250453:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000100:00000200:1.0:1775767875.250457:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dff100 x1862022294245760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767900 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1775767875.250465:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:2.0:1775767875.250475:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:2.0:1775767875.250489:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767875.250500:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767875.250508:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767875.250517:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:1.0:1775767875.250521:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000020:00000040:2.0:1775767875.250521:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00000040:1.0:1775767875.250530:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dff100 x1862022294245760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767900 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767875.250535:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775767875.250549:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767875.250550:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767875.250559:0:2399:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:2.0:1775767875.250559:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775767875.250567:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775767875.250568:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767875.250579:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767875.250588:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:1.0:1775767875.250589:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00100000:2.0:1775767875.250591:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1775767875.250596:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299801406 00000008:00000001:2.0:1775767875.250600:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767875.250607:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775767875.250611:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba180. 00000100:00000010:2.0:1775767875.250614:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfc700. 00000100:00000001:1.0:1775767875.250619:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000020:00000040:2.0:1775767875.250626:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000100:00000001:1.0:1775767875.250629:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299801406 02000000:00000001:2.0:1775767875.250635:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000010:1.0:1775767875.250638:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cbaf60. 02000000:00000001:2.0:1775767875.250639:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767875.250647:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1775767875.250649:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 02000000:00000001:2.0:1775767875.250650:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1775767875.250656:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767875.250658:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767875.250660:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767875.250668:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 02000000:00000010:2.0:1775767875.250668:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1e400. 00000100:00000001:2.0:1775767875.250674:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767875.250677:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767875/real 1775767875] req@ffff994207dff100 x1862022294245760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767875.250685:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfc700 x1862022294246016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:1.0:1775767875.250735:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1775767875.250737:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000001:1.0:1775767875.250743:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1775767875.250743:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775767875.250750:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023cf78. 00000100:00000040:2.0:1775767875.250756:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a176dd1] to pc [ptlrpcd_rcv+-1] req@ffff994207dfc700 x1862022294246016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775767875.250760:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf63b8 00000400:00000010:1.0:1775767875.250767:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf63b8. 00000100:00000001:1.0:1775767875.250776:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775767875.250793:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dff100 x1862022294245760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767875.250795:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775767875.250806:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767875.250824:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1775767875.250830:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775767875.250838:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:1.0:1775767875.250849:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dff100 x1862022294245760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767875.250850:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775767875.250861:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5127 00000100:00000001:1.0:1775767875.250877:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:2.0:1775767875.250880:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767875.250881:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767875.250890:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000400:00000200:2.0:1775767875.250894:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:1.0:1775767875.250895:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1775767875.250903:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775767875.250906:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dff100 x1862022294245760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775767875.250918:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767875.250934:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775767875.250935:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767875.250942:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767875.250951:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000040:1.0:1775767875.250952:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dff100 x1862022294245760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767900 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1775767875.250960:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775767875.250965:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:2.0:1775767875.250973:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775767875.250978:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:2.0:1775767875.250986:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:1.0:1775767875.250988:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:2.0:1775767875.250994:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:1.0:1775767875.250995:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767875.251001:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767875.251005:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767875.251009:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767875.251013:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767875.251018:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775767875.251021:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:2.0:1775767875.251026:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00080000:1.0:1775767875.251033:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:2.0:1775767875.251034:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775767875.251044:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:1.0:1775767875.251052:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00080000:2.0:1775767875.251053:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775767875.251065:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767875.251066:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:2.0:1775767875.251072:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:1.0:1775767875.251077:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dff100 x1862022294245760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767900 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1775767875.251079:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767875.251092:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767875.251117:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775767875.251123:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000100:00100000:1.0:1775767875.251126:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294245760:192.168.202.156@tcp:250:kworker.0 00000008:00000001:2.0:1775767875.251135:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767875.251138:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767875.251143:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767875.251144:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767875.251152:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dff100 x1862022294245760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767900 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1775767875.251191:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfe300. 00000020:00000040:2.0:1775767875.251201:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:2.0:1775767875.251213:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1775767875.251217:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767875.251217:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767875.251225:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775767875.251226:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1775767875.251229:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000010:1.0:1775767875.251232:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543dc00. 02000000:00000001:2.0:1775767875.251233:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767875.251244:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000010:2.0:1775767875.251245:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1f400. 00000020:00000001:1.0:1775767875.251253:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:2.0:1775767875.251254:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1775767875.251257:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:1.0:1775767875.251266:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00100000:2.0:1775767875.251272:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfe300 x1862022294246144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000010:1.0:1775767875.251273:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1d000. 02000000:00000001:1.0:1775767875.251287:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775767875.251291:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775767875.251296:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dff100. 00000100:00000001:1.0:1775767875.251307:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767875.251309:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775767875.251310:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767875.251317:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767875.251318:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767875.251332:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:2.0:1775767875.251333:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c661fef] to pc [ptlrpcd_rcv+-1] req@ffff994207dfe300 x1862022294246144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767875.251337:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767875.251350:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775767875.251358:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767875.251365:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1775767875.251374:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767875.251377:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfdc00 x1862022294245888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767875.251382:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767875.251399:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1775767875.251407:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775767875.251409:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767875.251415:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775767875.251415:0:36:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:1.0:1775767875.251428:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1775767875.251428:0:36:0:(import.c:484:import_select_connection()) Process entered 02000000:00000001:1.0:1775767875.251435:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1775767875.251437:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5127 02000000:00000001:1.0:1775767875.251444:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767875.251451:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294245888:192.168.202.156@tcp:38:kworker.0 00000400:00000200:2.0:1775767875.251454:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767875.251466:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:1.0:1775767875.251472:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00080000:2.0:1775767875.251472:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 02000000:00000001:1.0:1775767875.251475:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775767875.251481:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767875.251484:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 02000000:00000001:1.0:1775767875.251486:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1775767875.251494:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000010:1.0:1775767875.251495:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543dc00. 00000100:00000040:2.0:1775767875.251499:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000001:1.0:1775767875.251503:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767875.251511:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775767875.251517:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000400:00000200:1.0:1775767875.251518:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000040:2.0:1775767875.251524:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000400:00000010:1.0:1775767875.251525:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023cf78. 00000020:00000001:2.0:1775767875.251533:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000010:1.0:1775767875.251535:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf63b8. 00000020:00000040:2.0:1775767875.251540:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000100:00000200:1.0:1775767875.251542:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294245888, portal 10 00000020:00000001:2.0:1775767875.251548:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000001:1.0:1775767875.251550:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775767875.251556:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611843584 : -112966097708032 : ffff994207dfdc00) 00000020:00000001:2.0:1775767875.251557:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:2.0:1775767875.251576:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000040:1.0:1775767875.251577:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfdc00 x1862022294245888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767900 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767875.251582:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767875.251590:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767875.251602:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775767875.251613:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00000001:1.0:1775767875.251620:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1775767875.251621:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775767875.251630:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767875.251631:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 02000000:00000001:2.0:1775767875.251640:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:1.0:1775767875.251642:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6dd0. 02000000:00000001:2.0:1775767875.251645:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000200:1.0:1775767875.251647:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294245888, offset 0 02000000:00000001:2.0:1775767875.251653:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767875.251656:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775767875.251662:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000400:00000200:1.0:1775767875.251663:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000008:00000001:2.0:1775767875.251669:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767875.251672:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767875.251679:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000010:2.0:1775767875.251682:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfed80. 00000400:00000200:1.0:1775767875.251688:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000020:00000040:2.0:1775767875.251688:0:36:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:2.0:1775767875.251698:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000200:1.0:1775767875.251702:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:2.0:1775767875.251702:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767875.251708:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775767875.251713:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:1.0:1775767875.251714:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 02000000:00000001:2.0:1775767875.251717:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767875.251725:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 02000000:00000010:2.0:1775767875.251726:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1fc00. 00000400:00000200:1.0:1775767875.251733:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00000001:2.0:1775767875.251739:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767875.251749:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225094:225094:256:4294967295] 192.168.202.156@tcp LPNI seq info [225090:225090:8:4294967295] 00000100:00100000:2.0:1775767875.251755:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfed80 x1862022294246272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775767875.251780:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767875.251821:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000100:00000001:2.0:1775767875.251836:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775767875.251846:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775767875.251847:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767875.251858:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541800. 00000100:00000040:2.0:1775767875.251858:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb9b9eae] to pc [ptlrpcd_rcv+-1] req@ffff994207dfed80 x1862022294246272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1775767875.251877:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775767875.251885:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775767875.251904:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767875.251912:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000100:1.0:1775767875.251929:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000100:00000001:2.0:1775767875.251931:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1775767875.251932:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541800. 00000400:00000200:1.0:1775767875.251945:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:2.0:1775767875.251946:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5137) 00000400:00000200:1.0:1775767875.251954:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775767875.251959:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6dd0 00000400:00000010:1.0:1775767875.251962:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6dd0. 00000100:00000001:1.0:1775767875.251965:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775767875.251971:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfdc00 x1862022294245888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767900 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767875.251983:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767875.251986:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfdc00 x1862022294245888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767900 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767875.252013:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767875.252017:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775767875.252023:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767875.252030:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:1.0:1775767875.252040:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767875.252045:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299800084 00000100:00000001:1.0:1775767875.252049:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299801406 00000100:00000001:1.0:1775767875.252055:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299800084 00000100:00000001:1.0:1775767875.252059:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299801406 00000100:00000001:1.0:1775767875.252062:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775767875.252066:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767875.252069:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775767875.252074:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767875/real 1775767875] req@ffff994207dfdc00 x1862022294245888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775767875.252090:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775767875.252092:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775767875.252096:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023cf78. 00000400:00000200:1.0:1775767875.252113:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf63b8 00000400:00000010:1.0:1775767875.252117:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf63b8. 00000100:00000001:1.0:1775767875.252123:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775767875.252129:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfdc00 x1862022294245888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775767875.252145:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfdc00 x1862022294245888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767875.252175:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775767875.252176:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767875.252178:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767875.252179:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767875.252183:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfdc00 x1862022294245888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767875.252190:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775767875.252197:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfdc00 x1862022294245888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767900 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767875.252210:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775767875.252214:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767875.252217:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767875.252219:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767875.252222:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775767875.252225:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775767875.252231:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767875.252237:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775767875.252241:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775767875.252247:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfdc00 x1862022294245888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767900 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767875.252267:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294245888:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775767875.252274:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775767875.252279:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfc700 x1862022294246016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767875.252294:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767875.252297:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767875.252300:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775767875.252303:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767875.252306:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767875.252311:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294246016:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775767875.252320:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775767875.252322:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775767875.252326:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767875.252331:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775767875.252336:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543d800. 02000000:00000001:1.0:1775767875.252340:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767875.252345:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775767875.252350:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023cf78. 00000400:00000010:1.0:1775767875.252355:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf63b8. 00000100:00000200:1.0:1775767875.252359:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294246016, portal 10 00000100:00000001:1.0:1775767875.252364:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775767875.252368:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611838208 : -112966097713408 : ffff994207dfc700) 00000100:00000040:1.0:1775767875.252377:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfc700 x1862022294246016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767900 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767875.252392:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775767875.252396:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775767875.252398:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6dd0. 00000100:00000200:1.0:1775767875.252403:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294246016, offset 0 00000400:00000200:1.0:1775767875.252410:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775767875.252415:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767875.252420:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775767875.252424:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775767875.252429:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775767875.252433:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775767875.252437:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775767875.252443:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225095:225095:256:4294967295] 192.168.202.156@tcp LPNI seq info [225091:225091:8:4294967295] 00000400:00000200:1.0:1775767875.252452:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767875.252463:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775767875.252472:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767875.252475:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541800. 00000800:00000200:1.0:1775767875.252482:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775767875.252488:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775767875.252494:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767875.252497:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000100:1.0:1775767875.252503:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767875.252505:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541800. 00000400:00000200:1.0:1775767875.252510:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775767875.252517:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775767875.252520:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6dd0 00000400:00000010:1.0:1775767875.252523:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6dd0. 00000100:00000001:1.0:1775767875.252528:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775767875.252532:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfc700 x1862022294246016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767900 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767875.252544:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767875.252551:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfc700 x1862022294246016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767900 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767875.252572:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767875.252575:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775767875.252580:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767875.252583:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:1.0:1775767875.252587:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767875.252589:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299800077 00000100:00000001:1.0:1775767875.252593:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299801406 00000100:00000001:1.0:1775767875.252596:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299800077 00000100:00000001:1.0:1775767875.252598:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299801406 00000100:00000001:1.0:1775767875.252601:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775767875.252603:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767875.252606:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775767875.252611:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767875/real 1775767875] req@ffff994207dfc700 x1862022294246016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775767875.252618:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775767875.252619:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775767875.252621:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023cf78. 00000400:00000200:1.0:1775767875.252624:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf63b8 00000400:00000010:1.0:1775767875.252625:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf63b8. 00000100:00000001:1.0:1775767875.252627:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775767875.252630:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfc700 x1862022294246016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775767875.252636:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfc700 x1862022294246016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767875.252641:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775767875.252643:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767875.252644:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767875.252645:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767875.252648:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfc700 x1862022294246016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767875.252653:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775767875.252655:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfc700 x1862022294246016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767900 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767875.252659:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775767875.252661:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767875.252662:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767875.252663:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767875.252664:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775767875.252665:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775767875.252667:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767875.252669:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775767875.252670:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775767875.252672:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfc700 x1862022294246016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767900 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767875.252677:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294246016:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775767875.252680:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767875.252682:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767875.252683:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfdc00 x1862022294245888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767900 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767875.252688:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775767875.252689:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775767875.252690:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543dc00. 02000000:00000001:1.0:1775767875.252692:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775767875.252693:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775767875.252695:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:1.0:1775767875.252697:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775767875.252699:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1f800. 02000000:00000001:1.0:1775767875.252701:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775767875.252703:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775767875.252704:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfdc00. 00000100:00000001:1.0:1775767875.252707:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775767875.252708:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767875.252709:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767875.252712:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfc700 x1862022294246016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767900 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767875.252719:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775767875.252720:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775767875.252722:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543d800. 02000000:00000001:1.0:1775767875.252724:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775767875.252725:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775767875.252727:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:1.0:1775767875.252729:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775767875.252730:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1e400. 02000000:00000001:1.0:1775767875.252733:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775767875.252734:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775767875.252736:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfc700. 00000100:00000001:1.0:1775767875.252738:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775767875.252739:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767875.252741:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767875.252745:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767875.252746:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767875.252748:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775767875.252751:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767875.252752:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775767875.252756:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfe300 x1862022294246144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767875.252762:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767875.252763:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767875.252765:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775767875.252766:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767875.252768:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767875.252770:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294246144:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775767875.252772:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775767875.252774:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775767875.252775:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767875.252777:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775767875.252779:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543d800. 02000000:00000001:1.0:1775767875.252781:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767875.252785:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775767875.252788:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023cf78. 00000400:00000010:1.0:1775767875.252791:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf63b8. 00000100:00000200:1.0:1775767875.252794:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294246144, portal 4 00000100:00000001:1.0:1775767875.252796:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775767875.252798:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611845376 : -112966097706240 : ffff994207dfe300) 00000100:00000040:1.0:1775767875.252803:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfe300 x1862022294246144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767900 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767875.252810:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775767875.252813:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775767875.252815:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6dd0. 00000100:00000200:1.0:1775767875.252818:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294246144, offset 0 00000400:00000200:1.0:1775767875.252822:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775767875.252827:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767875.252830:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775767875.252833:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775767875.252836:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775767875.252838:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775767875.252840:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775767875.252842:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225096:225096:256:4294967295] 192.168.202.156@tcp LPNI seq info [225092:225092:8:4294967295] 00000400:00000200:1.0:1775767875.252845:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767875.252850:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775767875.252853:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767875.252855:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541800. 00000800:00000200:1.0:1775767875.252858:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775767875.252860:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775767875.252863:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767875.252865:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000100:1.0:1775767875.252867:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767875.252868:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541800. 00000400:00000200:1.0:1775767875.252870:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775767875.252873:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775767875.252874:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6dd0 00000400:00000010:1.0:1775767875.252876:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6dd0. 00000100:00000001:1.0:1775767875.252877:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775767875.252880:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfe300 x1862022294246144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767900 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767875.252885:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767875.252887:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfe300 x1862022294246144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767900 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767875.252892:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767875.252893:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775767875.252895:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767875.252898:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:1.0:1775767875.252899:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767875.252901:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299800100 00000100:00000001:1.0:1775767875.252903:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299801406 00000100:00000001:1.0:1775767875.252904:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299800100 00000100:00000001:1.0:1775767875.252906:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299801406 00000100:00000001:1.0:1775767875.252908:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775767875.252909:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767875.252911:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775767875.252913:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767875/real 1775767875] req@ffff994207dfe300 x1862022294246144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775767875.252918:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775767875.252919:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775767875.252920:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023cf78. 00000400:00000200:1.0:1775767875.252921:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf63b8 00000400:00000010:1.0:1775767875.252922:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf63b8. 00000100:00000001:1.0:1775767875.252923:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775767875.252925:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfe300 x1862022294246144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775767875.252930:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfe300 x1862022294246144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767875.252935:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775767875.252935:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767875.252936:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767875.252937:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767875.252939:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfe300 x1862022294246144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767875.252943:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775767875.252945:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfe300 x1862022294246144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767900 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767875.252949:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775767875.252951:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767875.252952:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767875.252952:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767875.252954:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775767875.252957:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775767875.252959:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767875.252961:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775767875.252963:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775767875.252966:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfe300 x1862022294246144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767900 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767875.252971:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294246144:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775767875.252973:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775767875.252975:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfed80 x1862022294246272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767875.252979:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767875.252980:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767875.252982:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775767875.252983:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767875.252984:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767875.252986:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294246272:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775767875.252988:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775767875.252989:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775767875.252990:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767875.252991:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775767875.252993:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543dc00. 02000000:00000001:1.0:1775767875.252994:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767875.252996:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775767875.252997:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023cf78. 00000400:00000010:1.0:1775767875.252999:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf63b8. 00000100:00000200:1.0:1775767875.253002:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294246272, portal 4 00000100:00000001:1.0:1775767875.253004:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775767875.253005:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611848064 : -112966097703552 : ffff994207dfed80) 00000100:00000040:1.0:1775767875.253011:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfed80 x1862022294246272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767900 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767875.253018:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775767875.253021:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775767875.253023:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6dd0. 00000100:00000200:1.0:1775767875.253025:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294246272, offset 0 00000400:00000200:1.0:1775767875.253030:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775767875.253034:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767875.253038:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775767875.253041:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775767875.253043:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775767875.253046:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775767875.253048:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775767875.253052:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225097:225097:256:4294967295] 192.168.202.156@tcp LPNI seq info [225093:225093:8:4294967295] 00000400:00000200:1.0:1775767875.253056:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767875.253062:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775767875.253065:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767875.253068:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541800. 00000800:00000200:1.0:1775767875.253071:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775767875.253072:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775767875.253075:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767875.253077:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000100:1.0:1775767875.253079:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767875.253081:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541800. 00000400:00000200:1.0:1775767875.253083:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775767875.253085:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775767875.253087:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6dd0 00000400:00000010:1.0:1775767875.253088:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6dd0. 00000100:00000001:1.0:1775767875.253090:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775767875.253092:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfed80 x1862022294246272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767900 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767875.253098:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767875.253112:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfed80 x1862022294246272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767900 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767875.253122:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767875.253123:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775767875.253126:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767875.253128:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:1.0:1775767875.253131:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767875.253132:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299800103 00000100:00000001:1.0:1775767875.253134:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299801406 00000100:00000001:1.0:1775767875.253135:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299800103 00000100:00000001:1.0:1775767875.253136:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299801406 00000100:00000001:1.0:1775767875.253138:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775767875.253139:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767875.253140:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775767875.253143:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767875/real 1775767875] req@ffff994207dfed80 x1862022294246272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775767875.253147:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775767875.253148:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775767875.253150:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023cf78. 00000400:00000200:1.0:1775767875.253151:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf63b8 00000400:00000010:1.0:1775767875.253152:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf63b8. 00000100:00000001:1.0:1775767875.253154:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775767875.253171:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfed80 x1862022294246272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775767875.253176:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfed80 x1862022294246272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767875.253181:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775767875.253182:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767875.253183:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767875.253184:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767875.253192:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfed80 x1862022294246272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767875.253197:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775767875.253199:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfed80 x1862022294246272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767900 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767875.253207:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775767875.253209:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767875.253210:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767875.253210:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767875.253212:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775767875.253214:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775767875.253215:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767875.253218:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775767875.253219:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775767875.253221:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfed80 x1862022294246272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767900 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767875.253226:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294246272:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775767875.253229:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767875.253230:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767875.253232:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfe300 x1862022294246144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767900 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767875.253235:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775767875.253237:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775767875.253238:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543d800. 02000000:00000001:1.0:1775767875.253240:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775767875.253241:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775767875.253242:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:1.0:1775767875.253243:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775767875.253244:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1f400. 02000000:00000001:1.0:1775767875.253246:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775767875.253247:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775767875.253248:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfe300. 00000100:00000001:1.0:1775767875.253250:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775767875.253251:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767875.253252:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767875.253254:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfed80 x1862022294246272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767900 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767875.253258:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775767875.253259:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775767875.253260:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543dc00. 02000000:00000001:1.0:1775767875.253261:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775767875.253261:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775767875.253262:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:1.0:1775767875.253263:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775767875.253264:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1fc00. 02000000:00000001:1.0:1775767875.253265:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775767875.253266:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775767875.253266:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfed80. 00000100:00000001:1.0:1775767875.253268:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775767875.253268:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767875.253269:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767875.253272:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767875.253273:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767875.253274:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775767875.253275:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767875.313364:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775767875.313371:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775767875.313377:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775767876.273277:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767876.273277:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767876.273278:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767876.273290:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767876.273291:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767876.273292:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767876.273299:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767876.273299:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767876.273299:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767876.273310:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775767876.273312:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:2.0:1775767876.273313:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767876.273315:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767876.273320:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767876.273321:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767876.273322:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767876.273323:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767876.273330:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767876.273331:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767876.273333:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767876.273337:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767876.273337:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767876.273340:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767876.273343:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767876.273344:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767876.273347:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767876.273350:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767876.593261:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299801406, at 4299801408 00000100:00000010:1.0:1775767876.593278:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cbaf60. 00000100:00000001:1.0:1775767877.297239:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767877.297239:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767877.297247:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767877.297248:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767877.297252:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767877.297253:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767877.297259:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775767877.297259:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767877.297266:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767877.297266:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767877.297270:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767877.297270:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767877.297275:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767877.297278:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767877.297281:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767877.297284:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767877.297285:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767877.297287:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767877.361376:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775767877.361380:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775767877.361383:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775767878.321224:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767878.321230:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767878.321232:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767878.321232:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767878.321237:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:1.0:1775767878.321239:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767878.321240:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767878.321245:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1775767878.321245:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767878.321251:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767878.321251:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767878.321254:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767878.321255:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767878.321258:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767878.321261:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767878.321264:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767878.321268:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767878.321270:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767878.321294:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767878.321301:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767878.321303:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767878.321309:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775767878.321314:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767878.321318:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767878.321321:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767878.321324:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767878.321326:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767879.345254:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767879.345254:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767879.345261:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767879.345261:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767879.345264:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767879.345265:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767879.345271:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:1.0:1775767879.345272:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767879.345276:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767879.345277:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767879.345277:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767879.345281:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767879.345286:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767879.345286:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767879.345287:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767879.345293:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767879.345294:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767879.345307:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767879.345307:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767879.345308:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767879.345314:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767879.345316:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767879.345317:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767879.345322:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767879.345326:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767879.345329:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767879.345333:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767879.409337:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775767879.409341:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775767879.409344:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775767880.369259:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767880.369259:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767880.369266:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767880.369267:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767880.369268:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767880.369270:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767880.369275:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:1.0:1775767880.369276:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767880.369281:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767880.369283:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767880.369285:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767880.369287:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767880.369289:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767880.369294:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767880.369298:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767880.369301:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767880.369302:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767880.369305:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767880.369312:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767880.369313:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767880.369321:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767880.369331:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775767880.369339:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767880.369345:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767880.369356:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767880.369361:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767880.369368:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767880.369517:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775767880.369521:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767880.369525:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775767880.369533:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fe300 x1862022294246400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767880.369551:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767880.369553:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767880.369557:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775767880.369561:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767880.369565:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767880.369570:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294246400:192.168.202.156@tcp:250:kworker.0 00000100:00000001:1.0:1775767880.369577:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775767880.369581:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775767880.369583:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767880.369587:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775767880.369593:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543f000. 02000000:00000001:1.0:1775767880.369595:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767880.369604:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775767880.369610:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023cf78. 00000400:00000010:1.0:1775767880.369617:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf63b8. 00000100:00000200:1.0:1775767880.369624:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294246400, portal 25 00000100:00000001:1.0:1775767880.369627:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775767880.369630:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641205504 : -112966068346112 : ffff9942099fe300) 00000100:00000040:1.0:1775767880.369639:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fe300 x1862022294246400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767905 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767880.369652:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775767880.369658:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775767880.369663:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6dd0. 00000100:00000200:1.0:1775767880.369667:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294246400, offset 0 00000400:00000200:1.0:1775767880.369675:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775767880.369685:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767880.369691:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775767880.369695:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775767880.369698:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775767880.369701:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775767880.369704:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775767880.369707:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225098:225098:256:4294967295] 192.168.202.156@tcp LPNI seq info [225094:225094:8:4294967295] 00000400:00000200:1.0:1775767880.369711:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767880.369718:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775767880.369726:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767880.369732:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541800. 00000800:00000200:1.0:1775767880.369747:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775767880.369755:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775767880.369767:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767880.369774:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1775767880.369784:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767880.369787:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541800. 00000400:00000200:1.0:1775767880.369794:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775767880.369803:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775767880.369809:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6dd0 00000400:00000010:1.0:1775767880.369813:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6dd0. 00000100:00000001:1.0:1775767880.369819:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775767880.369825:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fe300 x1862022294246400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767905 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767880.369841:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767880.369846:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fe300 x1862022294246400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767905 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767880.369858:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767880.369862:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775767880.369869:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767880.369877:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767880.369880:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299806525 00000100:00000010:1.0:1775767880.369888:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cbaf60. 00000100:00000001:1.0:1775767880.369895:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775767880.369902:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299806525 00000100:00000010:1.0:1775767880.369907:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba840. 00000100:00000001:1.0:1775767880.369911:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775767880.369916:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775767880.369920:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767880.369924:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775767880.369931:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767880/real 1775767880] req@ffff9942099fe300 x1862022294246400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775767880.369944:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775767880.369946:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775767880.369949:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023cf78. 00000400:00000200:1.0:1775767880.369952:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf63b8 00000400:00000010:1.0:1775767880.369955:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf63b8. 00000100:00000001:1.0:1775767880.369958:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775767880.369962:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fe300 x1862022294246400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775767880.369977:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fe300 x1862022294246400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767880.369987:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775767880.369989:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767880.369992:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767880.369995:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767880.370003:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fe300 x1862022294246400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767880.370030:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775767880.370038:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fe300 x1862022294246400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767905 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767880.370061:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775767880.370064:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767880.370067:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767880.370069:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767880.370071:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775767880.370074:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775767880.370079:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767880.370086:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775767880.370090:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775767880.370097:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fe300 x1862022294246400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767905 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767880.370112:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294246400:192.168.202.156@tcp:250:kworker.0 00000100:00000001:1.0:1775767880.370120:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767880.370124:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767880.370128:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fe300 x1862022294246400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767905 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767880.370145:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775767880.370148:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775767880.370152:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543f000. 02000000:00000001:1.0:1775767880.370173:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775767880.370174:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775767880.370175:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:1.0:1775767880.370177:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775767880.370179:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028400. 02000000:00000001:1.0:1775767880.370183:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775767880.370185:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775767880.370187:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fe300. 00000100:00000001:1.0:1775767880.370192:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775767880.370194:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767880.370200:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767880.370207:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767880.370210:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767880.370214:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775767880.370216:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767880.370219:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775767880.370226:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ff100 x1862022294246528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767880.370242:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767880.370245:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767880.370249:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775767880.370252:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767880.370257:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767880.370263:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294246528:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775767880.370269:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775767880.370272:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775767880.370276:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767880.370281:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775767880.370287:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543f000. 02000000:00000001:1.0:1775767880.370292:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767880.370301:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775767880.370307:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023cf78. 00000400:00000010:1.0:1775767880.370315:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf63b8. 00000100:00000200:1.0:1775767880.370320:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294246528, portal 10 00000100:00000001:1.0:1775767880.370325:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775767880.370328:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641209088 : -112966068342528 : ffff9942099ff100) 00000100:00000040:1.0:1775767880.370340:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ff100 x1862022294246528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767905 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767880.370365:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775767880.370370:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775767880.370380:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6dd0. 00000100:00000200:1.0:1775767880.370384:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294246528, offset 0 00000400:00000200:1.0:1775767880.370387:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775767880.370391:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767880.370396:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775767880.370398:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775767880.370401:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775767880.370403:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775767880.370406:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775767880.370409:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225099:225099:256:4294967295] 192.168.202.156@tcp LPNI seq info [225095:225095:8:4294967295] 00000400:00000200:1.0:1775767880.370412:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767880.370417:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775767880.370421:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767880.370423:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541800. 00000800:00000200:1.0:1775767880.370426:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775767880.370428:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775767880.370432:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767880.370445:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1775767880.370448:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767880.370449:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541800. 00000400:00000200:1.0:1775767880.370452:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775767880.370454:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775767880.370457:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6dd0 00000400:00000010:1.0:1775767880.370458:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6dd0. 00000100:00000001:1.0:1775767880.370460:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775767880.370462:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ff100 x1862022294246528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767905 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767880.370468:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767880.370470:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ff100 x1862022294246528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767905 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767880.370475:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767880.370476:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775767880.370478:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767880.370481:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:1.0:1775767880.370483:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767880.370484:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299805203 00000100:00000001:1.0:1775767880.370486:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299806525 00000100:00000001:1.0:1775767880.370488:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299805203 00000100:00000001:1.0:1775767880.370489:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299806525 00000100:00000001:1.0:1775767880.370491:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775767880.370493:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767880.370494:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775767880.370496:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767880/real 1775767880] req@ffff9942099ff100 x1862022294246528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775767880.370502:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775767880.370503:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775767880.370504:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023cf78. 00000400:00000200:1.0:1775767880.370506:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf63b8 00000400:00000010:1.0:1775767880.370507:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf63b8. 00000100:00000001:1.0:1775767880.370508:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775767880.370510:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ff100 x1862022294246528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775767880.370515:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ff100 x1862022294246528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767880.370520:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775767880.370521:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767880.370522:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767880.370522:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767880.370525:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ff100 x1862022294246528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767880.370529:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775767880.370531:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ff100 x1862022294246528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767905 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767880.370535:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775767880.370536:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767880.370537:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767880.370538:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767880.370539:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775767880.370541:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775767880.370542:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767880.370545:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775767880.370546:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775767880.370549:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ff100 x1862022294246528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767905 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767880.370554:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294246528:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775767880.370557:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775767880.370558:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fc700 x1862022294246656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767880.370562:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767880.370563:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767880.370564:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775767880.370566:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767880.370567:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767880.370569:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294246656:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775767880.370570:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775767880.370572:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775767880.370573:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767880.370574:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775767880.370576:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543d400. 02000000:00000001:1.0:1775767880.370577:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767880.370579:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775767880.370581:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023cf78. 00000400:00000010:1.0:1775767880.370583:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf63b8. 00000100:00000200:1.0:1775767880.370585:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294246656, portal 10 00000100:00000001:1.0:1775767880.370586:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775767880.370587:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641198336 : -112966068353280 : ffff9942099fc700) 00000100:00000040:1.0:1775767880.370591:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fc700 x1862022294246656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767905 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767880.370595:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775767880.370596:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775767880.370598:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6dd0. 00000100:00000200:1.0:1775767880.370599:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294246656, offset 0 00000400:00000200:1.0:1775767880.370602:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775767880.370605:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767880.370607:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775767880.370609:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775767880.370611:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775767880.370613:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775767880.370614:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775767880.370625:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225100:225100:256:4294967295] 192.168.202.156@tcp LPNI seq info [225096:225096:8:4294967295] 00000400:00000200:1.0:1775767880.370628:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767880.370632:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775767880.370638:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767880.370640:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541800. 00000800:00000200:1.0:1775767880.370643:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775767880.370645:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775767880.370648:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767880.370649:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1775767880.370652:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767880.370653:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541800. 00000400:00000200:1.0:1775767880.370655:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775767880.370657:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775767880.370659:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6dd0 00000400:00000010:1.0:1775767880.370660:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6dd0. 00000100:00000001:1.0:1775767880.370662:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775767880.370664:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fc700 x1862022294246656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767905 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767880.370669:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767880.370671:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fc700 x1862022294246656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767905 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767880.370675:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767880.370676:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775767880.370678:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767880.370680:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:1.0:1775767880.370682:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767880.370683:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299805195 00000100:00000001:1.0:1775767880.370685:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299806525 00000100:00000001:1.0:1775767880.370687:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299805195 00000100:00000001:1.0:1775767880.370688:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299806525 00000100:00000001:1.0:1775767880.370689:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775767880.370690:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767880.370691:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775767880.370693:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767880/real 1775767880] req@ffff9942099fc700 x1862022294246656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775767880.370698:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775767880.370699:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775767880.370700:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023cf78. 00000400:00000200:1.0:1775767880.370701:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf63b8 00000400:00000010:1.0:1775767880.370702:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf63b8. 00000100:00000001:1.0:1775767880.370704:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775767880.370706:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fc700 x1862022294246656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775767880.370710:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fc700 x1862022294246656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767880.370714:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775767880.370716:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767880.370717:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767880.370718:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767880.370720:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fc700 x1862022294246656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767880.370724:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775767880.370726:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fc700 x1862022294246656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767905 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767880.370730:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775767880.370733:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767880.370734:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767880.370735:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767880.370737:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775767880.370740:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775767880.370743:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767880.370746:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775767880.370748:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775767880.370752:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fc700 x1862022294246656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767905 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767880.370760:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294246656:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775767880.370764:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775767880.370767:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fd500 x1862022294246784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767880.370775:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767880.370777:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767880.370779:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775767880.370781:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767880.370783:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767880.370786:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294246784:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775767880.370790:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775767880.370791:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775767880.370793:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767880.370795:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775767880.370797:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543e400. 02000000:00000001:1.0:1775767880.370799:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767880.370801:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775767880.370803:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023cf78. 00000400:00000010:1.0:1775767880.370805:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf63b8. 00000100:00000200:1.0:1775767880.370808:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294246784, portal 4 00000100:00000001:1.0:1775767880.370810:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775767880.370811:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641201920 : -112966068349696 : ffff9942099fd500) 00000100:00000040:1.0:1775767880.370814:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fd500 x1862022294246784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767905 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767880.370819:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775767880.370820:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775767880.370822:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6dd0. 00000100:00000200:1.0:1775767880.370823:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294246784, offset 0 00000400:00000200:1.0:1775767880.370826:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775767880.370828:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767880.370831:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775767880.370832:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775767880.370834:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775767880.370836:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775767880.370837:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775767880.370839:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225101:225101:256:4294967295] 192.168.202.156@tcp LPNI seq info [225097:225097:8:4294967295] 00000400:00000200:1.0:1775767880.370842:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767880.370846:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775767880.370849:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767880.370851:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541800. 00000800:00000200:1.0:1775767880.370854:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775767880.370855:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775767880.370858:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767880.370860:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1775767880.370862:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767880.370864:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541800. 00000400:00000200:1.0:1775767880.370866:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775767880.370868:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775767880.370870:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6dd0 00000400:00000010:1.0:1775767880.370871:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6dd0. 00000100:00000001:1.0:1775767880.370873:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775767880.370875:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fd500 x1862022294246784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767905 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767880.370884:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767880.370886:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fd500 x1862022294246784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767905 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767880.370890:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767880.370891:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775767880.370893:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767880.370895:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:1.0:1775767880.370897:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767880.370898:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299805218 00000100:00000001:1.0:1775767880.370899:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299806525 00000100:00000001:1.0:1775767880.370901:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299805218 00000100:00000001:1.0:1775767880.370902:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299806525 00000100:00000001:1.0:1775767880.370904:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775767880.370905:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767880.370906:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775767880.370908:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767880/real 1775767880] req@ffff9942099fd500 x1862022294246784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775767880.370913:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775767880.370914:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775767880.370915:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023cf78. 00000400:00000200:1.0:1775767880.370916:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf63b8 00000400:00000010:1.0:1775767880.370917:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf63b8. 00000100:00000001:1.0:1775767880.370918:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775767880.370920:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fd500 x1862022294246784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775767880.370925:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fd500 x1862022294246784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767880.370929:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775767880.370930:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767880.370931:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767880.370932:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767880.370934:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fd500 x1862022294246784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767880.370938:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775767880.370941:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fd500 x1862022294246784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767905 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767880.370945:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775767880.370946:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767880.370947:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767880.370948:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767880.370949:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775767880.370951:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775767880.370953:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767880.370955:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775767880.370956:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775767880.370958:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fd500 x1862022294246784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767905 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767880.370963:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294246784:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775767880.370965:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767880.370966:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767880.370968:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ff100 x1862022294246528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767905 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767880.370973:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775767880.370974:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775767880.370975:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543f000. 02000000:00000001:1.0:1775767880.370977:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775767880.370978:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775767880.370979:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:1.0:1775767880.370981:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775767880.370982:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029400. 02000000:00000001:1.0:1775767880.370984:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775767880.370984:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775767880.370985:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ff100. 00000100:00000001:1.0:1775767880.370987:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775767880.370988:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767880.370989:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767880.370991:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fc700 x1862022294246656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767905 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767880.370995:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775767880.370996:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775767880.370997:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543d400. 02000000:00000001:1.0:1775767880.370998:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775767880.370999:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775767880.370999:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:1.0:1775767880.371000:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775767880.371001:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002b800. 02000000:00000001:1.0:1775767880.371002:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775767880.371003:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775767880.371004:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fc700. 00000100:00000001:1.0:1775767880.371005:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775767880.371006:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767880.371007:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767880.371008:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fd500 x1862022294246784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767905 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767880.371012:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775767880.371013:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775767880.371014:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543e400. 02000000:00000001:1.0:1775767880.371015:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775767880.371015:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775767880.371016:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:1.0:1775767880.371017:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775767880.371018:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028000. 02000000:00000001:1.0:1775767880.371019:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775767880.371020:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775767880.371021:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fd500. 00000100:00000001:1.0:1775767880.371023:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775767880.371023:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767880.371025:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767880.371027:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767880.371028:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767880.371029:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775767880.371030:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767880.371031:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775767880.371033:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ff800 x1862022294246912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767880.371038:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767880.371039:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767880.371040:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775767880.371042:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767880.371043:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767880.371045:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294246912:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775767880.371046:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775767880.371047:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775767880.371048:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767880.371049:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775767880.371067:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543e400. 02000000:00000001:1.0:1775767880.371068:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767880.371071:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775767880.371072:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023cf78. 00000400:00000010:1.0:1775767880.371074:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf63b8. 00000100:00000200:1.0:1775767880.371076:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294246912, portal 4 00000100:00000001:1.0:1775767880.371078:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775767880.371079:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641210880 : -112966068340736 : ffff9942099ff800) 00000100:00000040:1.0:1775767880.371083:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ff800 x1862022294246912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767905 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767880.371087:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775767880.371089:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775767880.371090:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6dd0. 00000100:00000200:1.0:1775767880.371092:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294246912, offset 0 00000400:00000200:1.0:1775767880.371094:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775767880.371097:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767880.371099:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775767880.371101:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775767880.371103:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775767880.371105:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775767880.371106:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775767880.371108:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225102:225102:256:4294967295] 192.168.202.156@tcp LPNI seq info [225098:225098:8:4294967295] 00000400:00000200:1.0:1775767880.371111:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767880.371115:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775767880.371118:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767880.371120:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541800. 00000800:00000200:1.0:1775767880.371122:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775767880.371124:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775767880.371127:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767880.371128:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1775767880.371131:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767880.371132:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541800. 00000400:00000200:1.0:1775767880.371134:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775767880.371136:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775767880.371138:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6dd0 00000400:00000010:1.0:1775767880.371139:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6dd0. 00000100:00000001:1.0:1775767880.371141:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775767880.371143:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ff800 x1862022294246912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767905 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767880.371148:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767880.371150:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ff800 x1862022294246912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767905 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767880.371154:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767880.371173:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775767880.371176:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767880.371178:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:1.0:1775767880.371181:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767880.371182:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299805222 00000100:00000001:1.0:1775767880.371184:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299806525 00000100:00000001:1.0:1775767880.371185:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299805222 00000100:00000001:1.0:1775767880.371187:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299806525 00000100:00000001:1.0:1775767880.371188:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775767880.371189:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767880.371190:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775767880.371193:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767880/real 1775767880] req@ffff9942099ff800 x1862022294246912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775767880.371198:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775767880.371199:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775767880.371200:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023cf78. 00000400:00000200:1.0:1775767880.371202:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf63b8 00000400:00000010:1.0:1775767880.371203:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf63b8. 00000100:00000001:1.0:1775767880.371204:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775767880.371206:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ff800 x1862022294246912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775767880.371211:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ff800 x1862022294246912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767880.371215:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775767880.371216:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767880.371217:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767880.371218:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767880.371220:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ff800 x1862022294246912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767880.371224:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775767880.371226:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ff800 x1862022294246912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767905 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767880.371231:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775767880.371233:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767880.371234:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767880.371234:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767880.371236:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775767880.371238:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775767880.371239:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767880.371241:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775767880.371243:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775767880.371245:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ff800 x1862022294246912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767905 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767880.371249:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294246912:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775767880.371252:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767880.371253:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767880.371255:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ff800 x1862022294246912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767905 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767880.371259:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775767880.371260:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775767880.371261:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543e400. 02000000:00000001:1.0:1775767880.371262:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775767880.371263:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775767880.371264:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:1.0:1775767880.371265:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775767880.371266:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002a800. 02000000:00000001:1.0:1775767880.371268:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775767880.371269:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775767880.371270:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ff800. 00000100:00000001:1.0:1775767880.371271:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775767880.371273:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767880.371275:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767880.371278:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767880.371279:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767880.371281:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775767880.371283:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767881.393240:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767881.393242:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767881.393251:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767881.393251:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767881.393254:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767881.393255:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767881.393262:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:1.0:1775767881.393265:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775767881.393265:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767881.393267:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767881.393269:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767881.393271:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1775767881.393273:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767881.393275:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767881.393275:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767881.393277:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767881.393279:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767881.393280:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767881.393283:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767881.393285:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767881.393288:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767881.393288:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767881.393291:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767881.393293:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767881.393294:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767881.393298:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767881.393301:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767881.457356:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775767881.457360:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775767881.457363:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775767881.713282:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299806525, at 4299806528 00000100:00000010:1.0:1775767881.713300:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cba840. 00000100:00000001:3.0:1775767882.417229:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767882.417241:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767882.417244:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767882.417249:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767882.417249:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:3.0:1775767882.417251:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767882.417256:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767882.417257:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767882.417258:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767882.417258:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767882.417262:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767882.417265:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775767882.417267:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767882.417274:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767882.417275:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767882.417275:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767882.417278:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767882.417280:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767882.417286:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767882.417289:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767882.417291:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767882.417294:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767882.417297:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767882.417300:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767882.417300:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767882.417303:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767882.417306:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767883.441216:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767883.441221:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767883.441224:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767883.441229:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767883.441232:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767883.441234:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767883.441238:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767883.441242:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767883.441243:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767883.441244:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767883.441250:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767883.441250:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767883.441251:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767883.441258:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767883.441259:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:1.0:1775767883.441260:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767883.441264:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767883.441267:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767883.441270:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767883.441273:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767883.441277:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767883.441278:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767883.441283:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767883.441286:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767883.441287:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767883.441290:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767883.441294:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767883.505294:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775767883.505297:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775767883.505300:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775767884.465210:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767884.465216:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767884.465218:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767884.465223:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767884.465226:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767884.465230:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767884.465233:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767884.465235:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767884.465238:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767884.465321:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767884.465329:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767884.465331:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767884.465337:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775767884.465342:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767884.465346:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767884.465349:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767884.465351:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767884.465355:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767884.465400:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767884.465404:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767884.465406:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767884.465408:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775767884.465412:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767884.465415:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767884.465419:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767884.465420:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767884.465424:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767885.489211:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767885.489218:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767885.489221:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767885.489228:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767885.489233:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767885.489238:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767885.489242:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767885.489245:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767885.489252:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767885.489301:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767885.489302:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767885.489304:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767885.489311:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767885.489313:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767885.489319:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767885.489319:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:2.0:1775767885.489329:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775767885.489334:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767885.489336:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767885.489341:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767885.489342:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767885.489346:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767885.489352:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767885.489364:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767885.489367:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767885.489375:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767885.489383:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767885.489503:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775767885.489507:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767885.489510:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775767885.489519:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fc000 x1862022294247040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767885.489535:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767885.489538:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767885.489544:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775767885.489547:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767885.489550:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767885.489555:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294247040:192.168.202.156@tcp:250:kworker.0 00000100:00000001:1.0:1775767885.489561:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775767885.489568:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775767885.489572:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767885.489589:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775767885.489597:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543d400. 02000000:00000001:1.0:1775767885.489601:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767885.489612:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775767885.489620:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023cf78. 00000400:00000010:1.0:1775767885.489630:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf63b8. 00000100:00000200:1.0:1775767885.489638:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294247040, portal 25 00000100:00000001:1.0:1775767885.489643:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775767885.489645:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641196544 : -112966068355072 : ffff9942099fc000) 00000100:00000040:1.0:1775767885.489653:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fc000 x1862022294247040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767910 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767885.489668:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775767885.489675:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775767885.489681:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6dd0. 00000100:00000200:1.0:1775767885.489687:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294247040, offset 0 00000400:00000200:1.0:1775767885.489699:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775767885.489711:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767885.489723:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775767885.489732:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775767885.489736:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775767885.489739:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775767885.489742:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775767885.489746:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225103:225103:256:4294967295] 192.168.202.156@tcp LPNI seq info [225099:225099:8:4294967295] 00000400:00000200:1.0:1775767885.489750:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767885.489758:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775767885.489766:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767885.489773:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541800. 00000800:00000200:1.0:1775767885.489785:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775767885.489800:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1775767885.489813:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767885.489821:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 11 secs later) 00000800:00000100:1.0:1775767885.489834:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767885.489838:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541800. 00000400:00000200:1.0:1775767885.489847:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775767885.489860:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775767885.489870:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6dd0 00000400:00000010:1.0:1775767885.489875:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6dd0. 00000100:00000001:1.0:1775767885.489880:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775767885.489886:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fc000 x1862022294247040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767910 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767885.489912:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767885.489917:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fc000 x1862022294247040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767910 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767885.489928:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767885.489930:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775767885.489938:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767885.489945:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767885.489948:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299811645 00000100:00000010:1.0:1775767885.489955:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba840. 00000100:00000001:1.0:1775767885.489963:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775767885.489970:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299811645 00000100:00000010:1.0:1775767885.489974:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba8a0. 00000100:00000001:1.0:1775767885.489978:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775767885.489982:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775767885.489986:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767885.489990:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775767885.489999:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767885/real 1775767885] req@ffff9942099fc000 x1862022294247040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775767885.490041:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775767885.490045:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775767885.490051:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023cf78. 00000400:00000200:1.0:1775767885.490058:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf63b8 00000400:00000010:1.0:1775767885.490060:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf63b8. 00000100:00000001:1.0:1775767885.490066:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775767885.490072:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fc000 x1862022294247040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775767885.490084:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fc000 x1862022294247040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767885.490096:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775767885.490100:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767885.490103:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767885.490105:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767885.490110:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fc000 x1862022294247040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767885.490121:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775767885.490128:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fc000 x1862022294247040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767910 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767885.490138:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775767885.490144:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767885.490153:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767885.490176:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767885.490178:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775767885.490181:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775767885.490184:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767885.490193:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775767885.490196:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775767885.490205:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fc000 x1862022294247040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767910 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767885.490233:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294247040:192.168.202.156@tcp:250:kworker.0 00000100:00000001:1.0:1775767885.490246:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767885.490252:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767885.490261:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fc000 x1862022294247040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767910 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767885.490284:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775767885.490290:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775767885.490293:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543d400. 02000000:00000001:1.0:1775767885.490297:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775767885.490301:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775767885.490303:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:1.0:1775767885.490309:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775767885.490314:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002b800. 02000000:00000001:1.0:1775767885.490319:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775767885.490322:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775767885.490324:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fc000. 00000100:00000001:1.0:1775767885.490329:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775767885.490331:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767885.490334:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767885.490342:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767885.490345:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767885.490349:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775767885.490351:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767885.490354:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775767885.490359:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ff800 x1862022294247168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767885.490371:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767885.490373:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767885.490376:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775767885.490379:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767885.490384:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767885.490388:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294247168:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775767885.490394:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775767885.490397:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775767885.490399:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767885.490402:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775767885.490405:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543d400. 02000000:00000001:1.0:1775767885.490408:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767885.490416:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775767885.490421:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023cf78. 00000400:00000010:1.0:1775767885.490425:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf63b8. 00000100:00000200:1.0:1775767885.490428:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294247168, portal 10 00000100:00000001:1.0:1775767885.490431:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775767885.490435:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641210880 : -112966068340736 : ffff9942099ff800) 00000100:00000040:1.0:1775767885.490442:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ff800 x1862022294247168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767910 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767885.490451:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775767885.490455:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775767885.490459:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6dd0. 00000100:00000200:1.0:1775767885.490463:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294247168, offset 0 00000400:00000200:1.0:1775767885.490473:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775767885.490480:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767885.490486:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775767885.490491:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775767885.490496:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775767885.490501:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775767885.490507:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775767885.490512:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225104:225104:256:4294967295] 192.168.202.156@tcp LPNI seq info [225100:225100:8:4294967295] 00000400:00000200:1.0:1775767885.490519:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767885.490526:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775767885.490530:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767885.490533:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541800. 00000800:00000200:1.0:1775767885.490537:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775767885.490540:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1775767885.490544:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767885.490546:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 11 secs later) 00000800:00000100:1.0:1775767885.490550:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767885.490552:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541800. 00000400:00000200:1.0:1775767885.490554:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775767885.490557:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775767885.490560:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6dd0 00000400:00000010:1.0:1775767885.490562:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6dd0. 00000100:00000001:1.0:1775767885.490565:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775767885.490569:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ff800 x1862022294247168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767910 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767885.490578:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767885.490581:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ff800 x1862022294247168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767910 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767885.490588:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767885.490590:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775767885.490593:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767885.490597:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:1.0:1775767885.490601:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767885.490603:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299810323 00000100:00000001:1.0:1775767885.490607:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299811645 00000100:00000001:1.0:1775767885.490609:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299810323 00000100:00000001:1.0:1775767885.490611:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299811645 00000100:00000001:1.0:1775767885.490613:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775767885.490616:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767885.490618:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775767885.490622:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767885/real 1775767885] req@ffff9942099ff800 x1862022294247168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775767885.490636:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775767885.490637:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775767885.490638:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023cf78. 00000400:00000200:1.0:1775767885.490640:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf63b8 00000400:00000010:1.0:1775767885.490641:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf63b8. 00000100:00000001:1.0:1775767885.490643:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775767885.490645:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ff800 x1862022294247168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775767885.490650:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ff800 x1862022294247168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767885.490654:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775767885.490655:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767885.490656:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767885.490657:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767885.490659:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ff800 x1862022294247168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767885.490664:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775767885.490666:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ff800 x1862022294247168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767910 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767885.490672:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775767885.490674:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767885.490676:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767885.490677:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767885.490679:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775767885.490682:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775767885.490685:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767885.490688:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775767885.490690:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775767885.490695:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ff800 x1862022294247168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767910 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767885.490703:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294247168:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775767885.490708:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775767885.490711:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fd500 x1862022294247296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767885.490719:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767885.490720:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767885.490723:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775767885.490724:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767885.490726:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767885.490729:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294247296:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775767885.490733:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775767885.490734:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775767885.490735:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767885.490737:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775767885.490739:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543f000. 02000000:00000001:1.0:1775767885.490740:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767885.490743:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775767885.490744:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023cf78. 00000400:00000010:1.0:1775767885.490747:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf63b8. 00000100:00000200:1.0:1775767885.490749:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294247296, portal 10 00000100:00000001:1.0:1775767885.490750:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775767885.490751:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641201920 : -112966068349696 : ffff9942099fd500) 00000100:00000040:1.0:1775767885.490755:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fd500 x1862022294247296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767910 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767885.490760:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775767885.490761:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775767885.490762:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6dd0. 00000100:00000200:1.0:1775767885.490764:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294247296, offset 0 00000400:00000200:1.0:1775767885.490767:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775767885.490770:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767885.490772:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775767885.490775:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775767885.490776:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775767885.490778:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775767885.490780:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775767885.490782:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225105:225105:256:4294967295] 192.168.202.156@tcp LPNI seq info [225101:225101:8:4294967295] 00000400:00000200:1.0:1775767885.490785:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767885.490789:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775767885.490792:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767885.490794:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541800. 00000800:00000200:1.0:1775767885.490797:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775767885.490798:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1775767885.490801:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767885.490803:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 11 secs later) 00000800:00000100:1.0:1775767885.490805:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767885.490807:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541800. 00000400:00000200:1.0:1775767885.490809:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775767885.490811:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775767885.490813:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6dd0 00000400:00000010:1.0:1775767885.490814:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6dd0. 00000100:00000001:1.0:1775767885.490816:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775767885.490818:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fd500 x1862022294247296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767910 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767885.490823:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767885.490825:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fd500 x1862022294247296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767910 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767885.490829:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767885.490830:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775767885.490832:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767885.490834:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:1.0:1775767885.490836:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767885.490838:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299810315 00000100:00000001:1.0:1775767885.490839:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299811645 00000100:00000001:1.0:1775767885.490841:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299810315 00000100:00000001:1.0:1775767885.490842:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299811645 00000100:00000001:1.0:1775767885.490843:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775767885.490844:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767885.490846:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775767885.490848:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767885/real 1775767885] req@ffff9942099fd500 x1862022294247296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775767885.490853:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775767885.490854:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775767885.490855:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023cf78. 00000400:00000200:1.0:1775767885.490856:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf63b8 00000400:00000010:1.0:1775767885.490857:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf63b8. 00000100:00000001:1.0:1775767885.490858:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775767885.490860:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fd500 x1862022294247296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775767885.490866:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fd500 x1862022294247296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767885.490870:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775767885.490871:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767885.490872:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767885.490873:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767885.490875:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fd500 x1862022294247296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767885.490879:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775767885.490881:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fd500 x1862022294247296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767910 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767885.490885:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775767885.490887:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767885.490888:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767885.490889:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767885.490890:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775767885.490892:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775767885.490894:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767885.490896:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775767885.490897:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775767885.490900:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fd500 x1862022294247296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767910 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767885.490904:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294247296:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775767885.490907:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775767885.490909:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fc700 x1862022294247424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767885.490913:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767885.490914:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767885.490915:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775767885.490916:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767885.490917:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767885.490919:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294247424:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775767885.490921:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775767885.490922:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775767885.490923:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767885.490924:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775767885.490926:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543d000. 02000000:00000001:1.0:1775767885.490927:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767885.490930:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775767885.490931:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023cf78. 00000400:00000010:1.0:1775767885.490933:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf63b8. 00000100:00000200:1.0:1775767885.490935:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294247424, portal 4 00000100:00000001:1.0:1775767885.490937:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775767885.490938:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641198336 : -112966068353280 : ffff9942099fc700) 00000100:00000040:1.0:1775767885.490941:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fc700 x1862022294247424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767910 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767885.490945:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775767885.490946:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775767885.490947:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6dd0. 00000100:00000200:1.0:1775767885.490949:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294247424, offset 0 00000400:00000200:1.0:1775767885.490951:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775767885.490953:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767885.490955:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775767885.490956:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775767885.490958:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775767885.490959:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775767885.490961:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775767885.490963:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225106:225106:256:4294967295] 192.168.202.156@tcp LPNI seq info [225102:225102:8:4294967295] 00000400:00000200:1.0:1775767885.490966:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767885.490970:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775767885.490972:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767885.490974:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541800. 00000800:00000200:1.0:1775767885.490976:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775767885.490978:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1775767885.490980:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767885.490982:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 11 secs later) 00000800:00000100:1.0:1775767885.490984:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767885.490986:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541800. 00000400:00000200:1.0:1775767885.490987:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775767885.490990:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775767885.490991:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6dd0 00000400:00000010:1.0:1775767885.490993:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6dd0. 00000100:00000001:1.0:1775767885.490995:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775767885.490998:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fc700 x1862022294247424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767910 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767885.491028:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767885.491032:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fc700 x1862022294247424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767910 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767885.491038:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767885.491040:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775767885.491043:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767885.491046:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:1.0:1775767885.491050:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767885.491052:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299810338 00000100:00000001:1.0:1775767885.491054:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299811645 00000100:00000001:1.0:1775767885.491056:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299810338 00000100:00000001:1.0:1775767885.491059:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299811645 00000100:00000001:1.0:1775767885.491061:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775767885.491063:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767885.491065:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775767885.491068:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767885/real 1775767885] req@ffff9942099fc700 x1862022294247424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775767885.491073:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775767885.491074:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775767885.491075:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023cf78. 00000400:00000200:1.0:1775767885.491077:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf63b8 00000400:00000010:1.0:1775767885.491078:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf63b8. 00000100:00000001:1.0:1775767885.491080:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775767885.491082:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fc700 x1862022294247424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775767885.491087:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fc700 x1862022294247424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767885.491091:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775767885.491092:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767885.491094:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767885.491095:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767885.491097:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fc700 x1862022294247424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767885.491101:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775767885.491103:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fc700 x1862022294247424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767910 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767885.491107:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775767885.491109:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767885.491109:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767885.491110:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767885.491111:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775767885.491114:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775767885.491115:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767885.491117:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775767885.491118:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775767885.491120:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fc700 x1862022294247424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767910 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767885.491125:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294247424:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775767885.491127:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767885.491128:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767885.491130:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ff800 x1862022294247168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767910 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767885.491135:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775767885.491136:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775767885.491137:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543d400. 02000000:00000001:1.0:1775767885.491139:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775767885.491140:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775767885.491141:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:1.0:1775767885.491142:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775767885.491143:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029400. 02000000:00000001:1.0:1775767885.491145:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775767885.491146:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775767885.491147:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ff800. 00000100:00000001:1.0:1775767885.491149:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775767885.491150:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767885.491151:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767885.491153:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fd500 x1862022294247296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767910 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767885.491185:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775767885.491186:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775767885.491187:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543f000. 02000000:00000001:1.0:1775767885.491189:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775767885.491190:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775767885.491190:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:1.0:1775767885.491192:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775767885.491193:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028400. 02000000:00000001:1.0:1775767885.491194:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775767885.491195:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775767885.491196:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fd500. 00000100:00000001:1.0:1775767885.491198:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775767885.491199:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767885.491200:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767885.491202:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fc700 x1862022294247424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767910 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767885.491206:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775767885.491208:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775767885.491209:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543d000. 02000000:00000001:1.0:1775767885.491211:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775767885.491212:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775767885.491213:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:1.0:1775767885.491215:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775767885.491216:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029800. 02000000:00000001:1.0:1775767885.491218:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775767885.491220:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775767885.491221:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fc700. 00000100:00000001:1.0:1775767885.491223:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775767885.491224:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767885.491226:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767885.491229:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767885.491230:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767885.491232:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775767885.491234:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767885.491235:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775767885.491238:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ff100 x1862022294247552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767885.491243:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767885.491244:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767885.491245:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775767885.491247:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767885.491248:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767885.491250:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294247552:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775767885.491251:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775767885.491253:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775767885.491254:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767885.491255:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775767885.491257:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543d000. 02000000:00000001:1.0:1775767885.491259:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767885.491261:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775767885.491262:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023cf78. 00000400:00000010:1.0:1775767885.491264:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf63b8. 00000100:00000200:1.0:1775767885.491266:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294247552, portal 4 00000100:00000001:1.0:1775767885.491268:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775767885.491269:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641209088 : -112966068342528 : ffff9942099ff100) 00000100:00000040:1.0:1775767885.491272:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ff100 x1862022294247552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767910 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767885.491276:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775767885.491278:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775767885.491280:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6dd0. 00000100:00000200:1.0:1775767885.491281:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294247552, offset 0 00000400:00000200:1.0:1775767885.491284:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775767885.491286:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767885.491289:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775767885.491291:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775767885.491293:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775767885.491294:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775767885.491296:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775767885.491298:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225107:225107:256:4294967295] 192.168.202.156@tcp LPNI seq info [225103:225103:8:4294967295] 00000400:00000200:1.0:1775767885.491300:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767885.491305:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775767885.491307:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767885.491309:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541800. 00000800:00000200:1.0:1775767885.491312:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775767885.491313:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1775767885.491316:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767885.491318:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 11 secs later) 00000800:00000100:1.0:1775767885.491320:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767885.491322:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541800. 00000400:00000200:1.0:1775767885.491323:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775767885.491326:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775767885.491327:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6dd0 00000400:00000010:1.0:1775767885.491329:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6dd0. 00000100:00000001:1.0:1775767885.491330:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775767885.491332:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ff100 x1862022294247552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767910 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767885.491337:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767885.491339:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ff100 x1862022294247552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767910 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767885.491343:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767885.491344:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775767885.491346:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767885.491348:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:1.0:1775767885.491350:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767885.491352:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299810342 00000100:00000001:1.0:1775767885.491353:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299811645 00000100:00000001:1.0:1775767885.491355:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299810342 00000100:00000001:1.0:1775767885.491356:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299811645 00000100:00000001:1.0:1775767885.491358:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775767885.491359:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767885.491361:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775767885.491363:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767885/real 1775767885] req@ffff9942099ff100 x1862022294247552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775767885.491367:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775767885.491368:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775767885.491368:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023cf78. 00000400:00000200:1.0:1775767885.491370:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf63b8 00000400:00000010:1.0:1775767885.491371:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf63b8. 00000100:00000001:1.0:1775767885.491372:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775767885.491374:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ff100 x1862022294247552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775767885.491379:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ff100 x1862022294247552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767885.491383:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775767885.491384:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767885.491385:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767885.491386:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767885.491388:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ff100 x1862022294247552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767885.491391:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775767885.491394:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ff100 x1862022294247552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767910 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767885.491399:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775767885.491400:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767885.491401:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767885.491402:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767885.491403:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775767885.491405:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775767885.491407:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767885.491409:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775767885.491410:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775767885.491412:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ff100 x1862022294247552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767910 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767885.491417:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294247552:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775767885.491419:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767885.491421:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767885.491422:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ff100 x1862022294247552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767910 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767885.491426:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775767885.491427:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775767885.491428:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543d000. 02000000:00000001:1.0:1775767885.491429:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775767885.491430:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775767885.491431:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:1.0:1775767885.491432:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775767885.491433:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002bc00. 02000000:00000001:1.0:1775767885.491434:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775767885.491435:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775767885.491436:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ff100. 00000100:00000001:1.0:1775767885.491438:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775767885.491438:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767885.491440:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767885.491441:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767885.491442:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767885.491444:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775767885.491445:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767885.553417:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775767885.553421:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775767885.553424:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775767886.513214:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767886.513216:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767886.513221:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767886.513221:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767886.513224:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767886.513224:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767886.513231:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775767886.513231:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775767886.513236:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767886.513236:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767886.513240:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767886.513240:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767886.513244:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767886.513246:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767886.513249:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767886.513250:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767886.513253:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767886.513255:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767886.833235:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299811645, at 4299811648 00000100:00000010:1.0:1775767886.833251:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cba8a0. 00000100:00000001:3.0:1775767887.537200:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767887.537205:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767887.537208:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767887.537212:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767887.537216:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767887.537219:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767887.537221:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767887.537225:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767887.537229:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767887.537235:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767887.537245:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767887.537249:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767887.537255:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775767887.537260:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767887.537264:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767887.537267:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767887.537269:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767887.537271:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767887.601364:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775767887.601368:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775767887.601370:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775767888.561239:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767888.561239:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767888.561245:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767888.561245:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767888.561249:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767888.561249:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767888.561256:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775767888.561257:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775767888.561262:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767888.561263:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767888.561268:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767888.561269:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767888.561269:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767888.561273:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767888.561274:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767888.561276:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767888.561277:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767888.561280:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767888.561281:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767888.561281:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767888.561283:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767888.561285:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767888.561288:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767888.561289:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767888.561294:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767888.561298:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767888.561301:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767889.585253:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767889.585253:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767889.585253:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767889.585260:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767889.585261:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767889.585261:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767889.585264:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767889.585264:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767889.585265:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767889.585271:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:2.0:1775767889.585271:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775767889.585272:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767889.585277:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767889.585277:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767889.585278:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767889.585282:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767889.585282:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767889.585283:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767889.585288:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767889.585291:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767889.585292:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767889.585295:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767889.585297:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767889.585299:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767889.585300:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767889.585304:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767889.585307:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767889.649294:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775767889.649297:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775767889.649300:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775767890.609233:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767890.609233:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767890.609239:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767890.609240:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767890.609240:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767890.609243:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767890.609244:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767890.609244:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767890.609247:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767890.609249:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775767890.609250:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:1.0:1775767890.609251:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767890.609253:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767890.609257:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767890.609258:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767890.609258:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767890.609261:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767890.609262:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767890.609263:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767890.609265:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767890.609268:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767890.609269:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767890.609272:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767890.609274:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767890.609275:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767890.609278:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767890.609281:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767890.609525:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775767890.609529:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767890.609532:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775767890.609541:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fe300 x1862022294247680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767890.609556:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767890.609558:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767890.609561:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775767890.609564:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767890.609567:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767890.609572:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294247680:192.168.202.156@tcp:250:kworker.0 00000100:00000001:1.0:1775767890.609578:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775767890.609582:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775767890.609584:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767890.609588:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775767890.609592:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543f000. 02000000:00000001:1.0:1775767890.609597:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767890.609605:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775767890.609609:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023cf78. 00000400:00000010:1.0:1775767890.609614:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf63b8. 00000100:00000200:1.0:1775767890.609621:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294247680, portal 25 00000100:00000001:1.0:1775767890.609626:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775767890.609629:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641205504 : -112966068346112 : ffff9942099fe300) 00000100:00000040:1.0:1775767890.609640:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fe300 x1862022294247680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767915 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767890.609659:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775767890.609664:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775767890.609670:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6dd0. 00000100:00000200:1.0:1775767890.609677:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294247680, offset 0 00000400:00000200:1.0:1775767890.609686:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775767890.609695:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767890.609702:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775767890.609708:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775767890.609713:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775767890.609719:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775767890.609723:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775767890.609742:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225108:225108:256:4294967295] 192.168.202.156@tcp LPNI seq info [225104:225104:8:4294967295] 00000400:00000200:1.0:1775767890.609751:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767890.609771:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775767890.609785:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767890.609791:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541800. 00000800:00000200:1.0:1775767890.609810:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775767890.609815:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1775767890.609825:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767890.609831:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000100:1.0:1775767890.609839:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767890.609844:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541800. 00000400:00000200:1.0:1775767890.609850:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775767890.609874:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775767890.609879:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6dd0 00000400:00000010:1.0:1775767890.609882:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6dd0. 00000100:00000001:1.0:1775767890.609888:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775767890.609894:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fe300 x1862022294247680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767915 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767890.609909:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767890.609914:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fe300 x1862022294247680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767915 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767890.609924:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767890.609928:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775767890.609933:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767890.609941:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767890.609945:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299816765 00000100:00000010:1.0:1775767890.609970:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba8a0. 00000100:00000001:1.0:1775767890.609978:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775767890.609983:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299816765 00000100:00000010:1.0:1775767890.609988:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cbab40. 00000100:00000001:1.0:1775767890.609992:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775767890.610013:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775767890.610018:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767890.610021:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775767890.610026:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767890/real 1775767890] req@ffff9942099fe300 x1862022294247680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775767890.610047:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775767890.610049:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775767890.610052:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023cf78. 00000400:00000200:1.0:1775767890.610056:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf63b8 00000400:00000010:1.0:1775767890.610059:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf63b8. 00000100:00000001:1.0:1775767890.610062:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775767890.610067:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fe300 x1862022294247680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775767890.610079:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fe300 x1862022294247680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767890.610091:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775767890.610094:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767890.610097:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767890.610099:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767890.610104:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fe300 x1862022294247680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767890.610116:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775767890.610123:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fe300 x1862022294247680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767915 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767890.610133:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775767890.610137:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767890.610141:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767890.610142:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767890.610145:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775767890.610148:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775767890.610154:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767890.610176:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775767890.610179:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775767890.610182:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fe300 x1862022294247680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767915 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767890.610189:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294247680:192.168.202.156@tcp:250:kworker.0 00000100:00000001:1.0:1775767890.610193:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767890.610194:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767890.610196:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fe300 x1862022294247680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767915 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767890.610205:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775767890.610208:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775767890.610211:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543f000. 02000000:00000001:1.0:1775767890.610215:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775767890.610217:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775767890.610221:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:1.0:1775767890.610223:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775767890.610226:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028400. 02000000:00000001:1.0:1775767890.610229:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775767890.610232:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775767890.610234:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fe300. 00000100:00000001:1.0:1775767890.610237:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775767890.610239:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767890.610244:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767890.610252:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767890.610255:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767890.610257:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775767890.610260:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767890.610262:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775767890.610267:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fd880 x1862022294247808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767890.610278:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767890.610281:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767890.610284:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775767890.610285:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767890.610289:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767890.610292:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294247808:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775767890.610297:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775767890.610299:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775767890.610300:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767890.610302:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775767890.610306:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543f000. 02000000:00000001:1.0:1775767890.610310:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767890.610317:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775767890.610319:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023cf78. 00000400:00000010:1.0:1775767890.610323:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf63b8. 00000100:00000200:1.0:1775767890.610326:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294247808, portal 10 00000100:00000001:1.0:1775767890.610329:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775767890.610331:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641202816 : -112966068348800 : ffff9942099fd880) 00000100:00000040:1.0:1775767890.610341:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fd880 x1862022294247808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767915 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767890.610351:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775767890.610355:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775767890.610356:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6dd0. 00000100:00000200:1.0:1775767890.610360:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294247808, offset 0 00000400:00000200:1.0:1775767890.610366:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775767890.610370:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767890.610375:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775767890.610382:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775767890.610389:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775767890.610391:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775767890.610394:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775767890.610397:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225109:225109:256:4294967295] 192.168.202.156@tcp LPNI seq info [225105:225105:8:4294967295] 00000400:00000200:1.0:1775767890.610410:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767890.610417:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775767890.610422:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767890.610429:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541800. 00000800:00000200:1.0:1775767890.610440:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775767890.610444:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1775767890.610452:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767890.610456:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000100:1.0:1775767890.610463:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767890.610467:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541800. 00000400:00000200:1.0:1775767890.610470:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775767890.610476:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775767890.610481:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6dd0 00000400:00000010:1.0:1775767890.610484:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6dd0. 00000100:00000001:1.0:1775767890.610487:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775767890.610492:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fd880 x1862022294247808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767915 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767890.610501:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767890.610506:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fd880 x1862022294247808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767915 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767890.610512:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767890.610514:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775767890.610517:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767890.610521:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:1.0:1775767890.610526:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767890.610529:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299815443 00000100:00000001:1.0:1775767890.610534:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299816765 00000100:00000001:1.0:1775767890.610537:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299815443 00000100:00000001:1.0:1775767890.610540:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299816765 00000100:00000001:1.0:1775767890.610543:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775767890.610549:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767890.610553:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775767890.610558:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767890/real 1775767890] req@ffff9942099fd880 x1862022294247808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775767890.610577:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775767890.610578:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775767890.610581:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023cf78. 00000400:00000200:1.0:1775767890.610583:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf63b8 00000400:00000010:1.0:1775767890.610586:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf63b8. 00000100:00000001:1.0:1775767890.610588:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775767890.610593:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fd880 x1862022294247808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775767890.610603:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fd880 x1862022294247808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767890.610610:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775767890.610612:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767890.610614:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767890.610616:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767890.610620:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fd880 x1862022294247808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767890.610639:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775767890.610649:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fd880 x1862022294247808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767915 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767890.610667:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775767890.610673:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767890.610677:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767890.610679:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767890.610682:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775767890.610684:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775767890.610688:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767890.610692:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775767890.610694:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775767890.610698:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fd880 x1862022294247808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767915 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767890.610704:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294247808:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775767890.610707:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775767890.610709:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ff480 x1862022294247936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767890.610714:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767890.610715:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767890.610716:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775767890.610718:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767890.610719:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767890.610721:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294247936:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775767890.610723:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775767890.610724:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775767890.610725:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767890.610726:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775767890.610729:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543d400. 02000000:00000001:1.0:1775767890.610730:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767890.610734:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775767890.610736:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023cf78. 00000400:00000010:1.0:1775767890.610739:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf63b8. 00000100:00000200:1.0:1775767890.610742:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294247936, portal 10 00000100:00000001:1.0:1775767890.610744:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775767890.610745:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641209984 : -112966068341632 : ffff9942099ff480) 00000100:00000040:1.0:1775767890.610751:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ff480 x1862022294247936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767915 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767890.610758:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775767890.610761:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775767890.610763:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6dd0. 00000100:00000200:1.0:1775767890.610766:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294247936, offset 0 00000400:00000200:1.0:1775767890.610771:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775767890.610775:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767890.610779:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775767890.610782:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775767890.610785:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775767890.610787:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775767890.610789:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775767890.610791:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225110:225110:256:4294967295] 192.168.202.156@tcp LPNI seq info [225106:225106:8:4294967295] 00000400:00000200:1.0:1775767890.610794:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767890.610799:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775767890.610802:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767890.610805:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541800. 00000800:00000200:1.0:1775767890.610808:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775767890.610809:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1775767890.610813:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767890.610814:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000100:1.0:1775767890.610817:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767890.610819:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541800. 00000400:00000200:1.0:1775767890.610821:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775767890.610823:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775767890.610825:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6dd0 00000400:00000010:1.0:1775767890.610826:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6dd0. 00000100:00000001:1.0:1775767890.610828:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775767890.610830:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ff480 x1862022294247936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767915 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767890.610835:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767890.610837:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ff480 x1862022294247936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767915 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767890.610842:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767890.610843:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775767890.610845:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767890.610847:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:1.0:1775767890.610849:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767890.610850:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299815435 00000100:00000001:1.0:1775767890.610852:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299816765 00000100:00000001:1.0:1775767890.610854:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299815435 00000100:00000001:1.0:1775767890.610855:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299816765 00000100:00000001:1.0:1775767890.610856:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775767890.610857:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767890.610859:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775767890.610861:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767890/real 1775767890] req@ffff9942099ff480 x1862022294247936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775767890.610865:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775767890.610866:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775767890.610868:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023cf78. 00000400:00000200:1.0:1775767890.610869:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf63b8 00000400:00000010:1.0:1775767890.610870:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf63b8. 00000100:00000001:1.0:1775767890.610872:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775767890.610873:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ff480 x1862022294247936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775767890.610879:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ff480 x1862022294247936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767890.610882:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775767890.610883:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767890.610884:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767890.610885:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767890.610887:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ff480 x1862022294247936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767890.610891:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775767890.610893:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ff480 x1862022294247936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767915 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767890.610897:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775767890.610899:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767890.610900:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767890.610901:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767890.610902:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775767890.610904:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775767890.610906:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767890.610908:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775767890.610909:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775767890.610911:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ff480 x1862022294247936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767915 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767890.610916:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294247936:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775767890.610918:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767890.610920:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767890.610922:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fd880 x1862022294247808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767915 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767890.610926:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775767890.610927:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775767890.610928:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543f000. 02000000:00000001:1.0:1775767890.610929:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775767890.610930:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775767890.610931:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:1.0:1775767890.610932:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775767890.610933:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029400. 02000000:00000001:1.0:1775767890.610935:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775767890.610936:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775767890.610937:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fd880. 00000100:00000001:1.0:1775767890.610939:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775767890.610939:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767890.610940:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767890.610942:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ff480 x1862022294247936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767915 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767890.610965:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775767890.610966:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775767890.610967:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543d400. 02000000:00000001:1.0:1775767890.610969:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775767890.610970:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775767890.610971:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:1.0:1775767890.610972:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775767890.610973:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002b800. 02000000:00000001:1.0:1775767890.610974:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775767890.610975:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775767890.610976:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ff480. 00000100:00000001:1.0:1775767890.610977:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775767890.610978:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767890.610979:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767890.610982:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767890.610983:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767890.610984:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775767890.610985:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767890.610986:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775767890.610989:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fd180 x1862022294248064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767890.610993:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767890.610995:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767890.610996:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775767890.610997:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767890.610998:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767890.611000:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294248064:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775767890.611002:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775767890.611003:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775767890.611004:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767890.611006:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775767890.611008:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543d400. 02000000:00000001:1.0:1775767890.611009:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767890.611012:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775767890.611014:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023cf78. 00000400:00000010:1.0:1775767890.611016:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf63b8. 00000100:00000200:1.0:1775767890.611018:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294248064, portal 4 00000100:00000001:1.0:1775767890.611019:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775767890.611020:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641201024 : -112966068350592 : ffff9942099fd180) 00000100:00000040:1.0:1775767890.611024:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fd180 x1862022294248064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767915 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767890.611028:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775767890.611030:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775767890.611031:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6dd0. 00000100:00000200:1.0:1775767890.611032:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294248064, offset 0 00000400:00000200:1.0:1775767890.611035:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775767890.611037:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767890.611039:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775767890.611041:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775767890.611043:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775767890.611045:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775767890.611046:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775767890.611048:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225111:225111:256:4294967295] 192.168.202.156@tcp LPNI seq info [225107:225107:8:4294967295] 00000400:00000200:1.0:1775767890.611051:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767890.611056:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775767890.611058:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767890.611060:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541800. 00000800:00000200:1.0:1775767890.611063:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775767890.611065:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1775767890.611068:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767890.611080:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000100:1.0:1775767890.611083:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767890.611085:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541800. 00000400:00000200:1.0:1775767890.611086:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775767890.611089:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775767890.611091:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6dd0 00000400:00000010:1.0:1775767890.611093:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6dd0. 00000100:00000001:1.0:1775767890.611095:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775767890.611097:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fd180 x1862022294248064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767915 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767890.611103:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767890.611104:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fd180 x1862022294248064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767915 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767890.611109:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767890.611109:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775767890.611111:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767890.611114:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:1.0:1775767890.611116:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767890.611117:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299815458 00000100:00000001:1.0:1775767890.611118:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299816765 00000100:00000001:1.0:1775767890.611120:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299815458 00000100:00000001:1.0:1775767890.611121:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299816765 00000100:00000001:1.0:1775767890.611123:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775767890.611124:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767890.611126:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775767890.611128:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767890/real 1775767890] req@ffff9942099fd180 x1862022294248064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775767890.611132:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775767890.611133:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775767890.611134:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023cf78. 00000400:00000200:1.0:1775767890.611135:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf63b8 00000400:00000010:1.0:1775767890.611136:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf63b8. 00000100:00000001:1.0:1775767890.611138:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775767890.611139:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fd180 x1862022294248064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775767890.611144:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fd180 x1862022294248064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767890.611149:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775767890.611150:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767890.611151:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767890.611152:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767890.611154:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fd180 x1862022294248064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767890.611173:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775767890.611176:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fd180 x1862022294248064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767915 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767890.611180:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775767890.611182:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767890.611183:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767890.611183:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767890.611184:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775767890.611187:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775767890.611189:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767890.611191:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775767890.611192:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775767890.611194:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fd180 x1862022294248064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767915 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767890.611198:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294248064:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775767890.611201:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775767890.611203:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fed80 x1862022294248192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767890.611207:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767890.611208:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767890.611210:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775767890.611212:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767890.611213:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767890.611215:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294248192:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775767890.611217:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775767890.611218:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775767890.611219:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767890.611220:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775767890.611222:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543f000. 02000000:00000001:1.0:1775767890.611223:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767890.611225:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775767890.611227:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023cf78. 00000400:00000010:1.0:1775767890.611229:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf63b8. 00000100:00000200:1.0:1775767890.611231:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294248192, portal 4 00000100:00000001:1.0:1775767890.611232:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775767890.611233:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641208192 : -112966068343424 : ffff9942099fed80) 00000100:00000040:1.0:1775767890.611237:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fed80 x1862022294248192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767915 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767890.611241:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775767890.611243:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775767890.611244:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6dd0. 00000100:00000200:1.0:1775767890.611245:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294248192, offset 0 00000400:00000200:1.0:1775767890.611248:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775767890.611250:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767890.611252:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775767890.611254:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775767890.611255:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775767890.611257:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775767890.611258:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775767890.611261:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225112:225112:256:4294967295] 192.168.202.156@tcp LPNI seq info [225108:225108:8:4294967295] 00000400:00000200:1.0:1775767890.611263:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767890.611267:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775767890.611271:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767890.611273:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541800. 00000800:00000200:1.0:1775767890.611275:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775767890.611277:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1775767890.611280:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767890.611281:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 6 secs later) 00000800:00000100:1.0:1775767890.611284:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767890.611285:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541800. 00000400:00000200:1.0:1775767890.611287:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775767890.611290:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775767890.611291:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6dd0 00000400:00000010:1.0:1775767890.611292:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6dd0. 00000100:00000001:1.0:1775767890.611294:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775767890.611296:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fed80 x1862022294248192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767915 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767890.611301:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767890.611303:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fed80 x1862022294248192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767915 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767890.611307:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767890.611308:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775767890.611310:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767890.611312:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:1.0:1775767890.611313:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767890.611315:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299815462 00000100:00000001:1.0:1775767890.611316:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299816765 00000100:00000001:1.0:1775767890.611318:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299815462 00000100:00000001:1.0:1775767890.611319:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299816765 00000100:00000001:1.0:1775767890.611320:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775767890.611322:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767890.611323:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775767890.611325:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767890/real 1775767890] req@ffff9942099fed80 x1862022294248192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775767890.611330:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775767890.611331:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775767890.611332:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023cf78. 00000400:00000200:1.0:1775767890.611334:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf63b8 00000400:00000010:1.0:1775767890.611334:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf63b8. 00000100:00000001:1.0:1775767890.611336:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775767890.611337:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fed80 x1862022294248192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775767890.611342:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fed80 x1862022294248192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767890.611346:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775767890.611347:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767890.611348:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767890.611349:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767890.611352:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fed80 x1862022294248192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767890.611356:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775767890.611358:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fed80 x1862022294248192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767915 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767890.611362:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775767890.611363:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767890.611364:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767890.611365:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767890.611366:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775767890.611368:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775767890.611369:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767890.611371:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775767890.611372:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775767890.611375:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fed80 x1862022294248192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767915 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767890.611382:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294248192:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775767890.611386:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767890.611387:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767890.611390:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fd180 x1862022294248064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767915 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767890.611397:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775767890.611399:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775767890.611400:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543d400. 02000000:00000001:1.0:1775767890.611403:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775767890.611404:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775767890.611406:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:1.0:1775767890.611408:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775767890.611409:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028000. 02000000:00000001:1.0:1775767890.611411:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775767890.611413:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775767890.611414:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fd180. 00000100:00000001:1.0:1775767890.611417:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775767890.611418:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767890.611420:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767890.611424:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fed80 x1862022294248192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767915 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767890.611431:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775767890.611433:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775767890.611434:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543f000. 02000000:00000001:1.0:1775767890.611436:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775767890.611437:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775767890.611438:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:1.0:1775767890.611440:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775767890.611442:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002a800. 02000000:00000001:1.0:1775767890.611444:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775767890.611445:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775767890.611446:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fed80. 00000100:00000001:1.0:1775767890.611447:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775767890.611448:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767890.611449:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767890.611452:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767890.611453:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767890.611454:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775767890.611455:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767891.633305:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767891.633305:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767891.633305:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767891.633313:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767891.633314:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767891.633315:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767891.633315:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767891.633317:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767891.633318:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767891.633325:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775767891.633325:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:2.0:1775767891.633326:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767891.633331:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767891.633332:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767891.633332:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767891.633337:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767891.633338:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767891.633338:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767891.633342:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767891.633346:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767891.633349:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767891.633352:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767891.633355:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767891.633357:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767891.633358:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767891.633362:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767891.633364:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767891.697383:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775767891.697387:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775767891.697390:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775767891.953280:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299816765, at 4299816768 00000100:00000010:1.0:1775767891.953304:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cbab40. 00000100:00000001:2.0:1775767892.657224:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767892.657224:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767892.657232:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767892.657232:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767892.657236:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767892.657237:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767892.657243:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:2.0:1775767892.657244:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775767892.657249:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767892.657249:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767892.657253:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767892.657254:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767892.657256:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767892.657260:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767892.657264:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767892.657265:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767892.657266:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767892.657272:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767892.657273:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767892.657274:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767892.657282:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767892.657283:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767892.657293:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767892.657298:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767892.657301:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767892.657303:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767892.657306:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767893.681264:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767893.681268:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767893.681272:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767893.681274:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767893.681279:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767893.681280:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767893.681284:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:1.0:1775767893.681290:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767893.681291:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:3.0:1775767893.681292:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775767893.681297:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767893.681300:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767893.681302:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:2.0:1775767893.681303:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767893.681305:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767893.681307:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767893.681309:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767893.681310:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767893.681311:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767893.681317:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767893.681320:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767893.681325:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767893.681329:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767893.681333:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767893.681333:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767893.681336:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767893.681341:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767894.705319:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767894.705320:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767894.705327:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767894.705327:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767894.705329:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767894.705331:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767894.705331:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767894.705339:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:2.0:1775767894.705341:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775767894.705341:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767894.705342:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767894.705352:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767894.705353:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1775767894.705354:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767894.705357:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767894.705357:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767894.705359:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767894.705366:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767894.705366:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767894.705371:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767894.705374:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767894.705375:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767894.705378:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767894.705381:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767894.705382:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767894.705386:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767894.705389:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767895.729266:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767895.729266:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767895.729275:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767895.729276:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767895.729278:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767895.729280:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767895.729283:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775767895.729284:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775767895.729292:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767895.729293:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767895.729297:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767895.729299:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767895.729299:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767895.729305:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767895.729306:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767895.729312:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767895.729312:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767895.729325:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767895.729325:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767895.729326:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767895.729338:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767895.729340:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767895.729348:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767895.729349:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767895.729352:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767895.729359:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767895.729363:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767895.729567:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775767895.729571:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767895.729575:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775767895.729584:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fc380 x1862022294248320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767895.729603:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767895.729605:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767895.729611:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775767895.729615:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767895.729619:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767895.729625:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294248320:192.168.202.156@tcp:250:kworker.0 00000100:00000001:1.0:1775767895.729631:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775767895.729635:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775767895.729638:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767895.729643:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775767895.729649:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543f000. 02000000:00000001:1.0:1775767895.729654:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767895.729662:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775767895.729669:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023cf78. 00000400:00000010:1.0:1775767895.729679:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf63b8. 00000100:00000200:1.0:1775767895.729688:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294248320, portal 25 00000100:00000001:1.0:1775767895.729694:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775767895.729696:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641197440 : -112966068354176 : ffff9942099fc380) 00000100:00000040:1.0:1775767895.729705:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fc380 x1862022294248320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767920 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767895.729726:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775767895.729733:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775767895.729736:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6dd0. 00000100:00000200:1.0:1775767895.729740:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294248320, offset 0 00000400:00000200:1.0:1775767895.729750:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775767895.729763:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767895.729773:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775767895.729779:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775767895.729785:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775767895.729792:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775767895.729807:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775767895.729814:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225113:225113:256:4294967295] 192.168.202.156@tcp LPNI seq info [225109:225109:8:4294967295] 00000400:00000200:1.0:1775767895.729823:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767895.729840:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775767895.729851:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767895.729858:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541800. 00000800:00000200:1.0:1775767895.729868:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775767895.729873:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775767895.729882:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767895.729887:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000100:1.0:1775767895.729896:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767895.729920:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541800. 00000400:00000200:1.0:1775767895.729929:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775767895.729938:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775767895.729947:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6dd0 00000400:00000010:1.0:1775767895.729950:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6dd0. 00000100:00000001:1.0:1775767895.729958:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775767895.729965:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fc380 x1862022294248320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767920 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767895.729977:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767895.729982:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fc380 x1862022294248320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767920 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767895.729996:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767895.729998:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775767895.730005:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767895.730013:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767895.730019:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299821885 00000100:00000010:1.0:1775767895.730025:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cbab40. 00000100:00000001:1.0:1775767895.730034:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775767895.730043:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299821885 00000100:00000010:1.0:1775767895.730048:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba540. 00000100:00000001:1.0:1775767895.730055:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775767895.730059:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775767895.730065:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767895.730069:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775767895.730076:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767895/real 1775767895] req@ffff9942099fc380 x1862022294248320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775767895.730086:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775767895.730089:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775767895.730091:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023cf78. 00000400:00000200:1.0:1775767895.730096:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf63b8 00000400:00000010:1.0:1775767895.730100:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf63b8. 00000100:00000001:1.0:1775767895.730103:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775767895.730109:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fc380 x1862022294248320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775767895.730123:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fc380 x1862022294248320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767895.730181:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775767895.730184:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767895.730189:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767895.730191:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767895.730200:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fc380 x1862022294248320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767895.730217:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775767895.730225:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fc380 x1862022294248320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767920 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767895.730239:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775767895.730244:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767895.730248:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767895.730250:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767895.730253:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775767895.730256:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775767895.730260:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767895.730269:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775767895.730274:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775767895.730282:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fc380 x1862022294248320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767920 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767895.730302:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294248320:192.168.202.156@tcp:250:kworker.0 00000100:00000001:1.0:1775767895.730310:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767895.730314:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767895.730319:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fc380 x1862022294248320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767920 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767895.730332:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775767895.730336:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775767895.730340:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543f000. 02000000:00000001:1.0:1775767895.730345:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775767895.730348:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775767895.730350:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:1.0:1775767895.730352:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775767895.730356:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029400. 02000000:00000001:1.0:1775767895.730363:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775767895.730366:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775767895.730368:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fc380. 00000100:00000001:1.0:1775767895.730373:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775767895.730375:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767895.730378:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767895.730387:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767895.730390:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767895.730395:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775767895.730397:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767895.730399:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775767895.730404:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fce00 x1862022294248448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767895.730410:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767895.730412:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767895.730414:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775767895.730416:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767895.730418:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767895.730420:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294248448:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775767895.730423:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775767895.730424:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775767895.730425:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767895.730427:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775767895.730429:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543f000. 02000000:00000001:1.0:1775767895.730431:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767895.730433:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775767895.730435:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023cf78. 00000400:00000010:1.0:1775767895.730438:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf63b8. 00000100:00000200:1.0:1775767895.730440:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294248448, portal 10 00000100:00000001:1.0:1775767895.730442:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775767895.730443:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641200128 : -112966068351488 : ffff9942099fce00) 00000100:00000040:1.0:1775767895.730448:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fce00 x1862022294248448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767920 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767895.730453:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775767895.730455:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775767895.730456:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6dd0. 00000100:00000200:1.0:1775767895.730458:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294248448, offset 0 00000400:00000200:1.0:1775767895.730460:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775767895.730463:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767895.730465:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775767895.730467:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775767895.730469:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775767895.730470:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775767895.730472:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775767895.730474:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225114:225114:256:4294967295] 192.168.202.156@tcp LPNI seq info [225110:225110:8:4294967295] 00000400:00000200:1.0:1775767895.730477:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767895.730482:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775767895.730485:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767895.730487:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541800. 00000800:00000200:1.0:1775767895.730489:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775767895.730491:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775767895.730494:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767895.730496:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000100:1.0:1775767895.730498:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767895.730499:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541800. 00000400:00000200:1.0:1775767895.730501:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775767895.730504:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775767895.730506:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6dd0 00000400:00000010:1.0:1775767895.730507:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6dd0. 00000100:00000001:1.0:1775767895.730509:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775767895.730511:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fce00 x1862022294248448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767920 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767895.730521:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767895.730523:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fce00 x1862022294248448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767920 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767895.730527:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767895.730528:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775767895.730530:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767895.730533:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:1.0:1775767895.730535:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767895.730536:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299820563 00000100:00000001:1.0:1775767895.730538:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299821885 00000100:00000001:1.0:1775767895.730539:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299820563 00000100:00000001:1.0:1775767895.730541:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299821885 00000100:00000001:1.0:1775767895.730543:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775767895.730544:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767895.730545:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775767895.730547:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767895/real 1775767895] req@ffff9942099fce00 x1862022294248448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775767895.730552:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775767895.730553:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775767895.730554:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023cf78. 00000400:00000200:1.0:1775767895.730555:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf63b8 00000400:00000010:1.0:1775767895.730556:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf63b8. 00000100:00000001:1.0:1775767895.730557:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775767895.730559:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fce00 x1862022294248448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775767895.730564:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fce00 x1862022294248448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767895.730568:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775767895.730570:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767895.730571:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767895.730572:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767895.730575:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fce00 x1862022294248448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767895.730579:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775767895.730581:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fce00 x1862022294248448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767920 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767895.730585:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775767895.730587:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767895.730588:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767895.730589:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767895.730590:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775767895.730592:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775767895.730593:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767895.730595:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775767895.730596:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775767895.730599:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fce00 x1862022294248448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767920 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767895.730603:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294248448:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775767895.730606:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775767895.730608:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fdc00 x1862022294248576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767895.730612:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767895.730613:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767895.730615:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775767895.730616:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767895.730617:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767895.730619:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294248576:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775767895.730621:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775767895.730623:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775767895.730624:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767895.730625:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775767895.730627:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543d400. 02000000:00000001:1.0:1775767895.730628:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767895.730630:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775767895.730631:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023cf78. 00000400:00000010:1.0:1775767895.730633:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf63b8. 00000100:00000200:1.0:1775767895.730635:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294248576, portal 10 00000100:00000001:1.0:1775767895.730637:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775767895.730638:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641203712 : -112966068347904 : ffff9942099fdc00) 00000100:00000040:1.0:1775767895.730642:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fdc00 x1862022294248576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767920 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767895.730646:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775767895.730647:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775767895.730649:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6dd0. 00000100:00000200:1.0:1775767895.730650:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294248576, offset 0 00000400:00000200:1.0:1775767895.730653:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775767895.730655:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767895.730656:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775767895.730658:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775767895.730659:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775767895.730661:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775767895.730662:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775767895.730664:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225115:225115:256:4294967295] 192.168.202.156@tcp LPNI seq info [225111:225111:8:4294967295] 00000400:00000200:1.0:1775767895.730667:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767895.730671:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775767895.730675:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767895.730676:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541800. 00000800:00000200:1.0:1775767895.730680:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775767895.730681:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775767895.730684:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767895.730685:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000100:1.0:1775767895.730688:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767895.730689:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541800. 00000400:00000200:1.0:1775767895.730691:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775767895.730694:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775767895.730695:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6dd0 00000400:00000010:1.0:1775767895.730696:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6dd0. 00000100:00000001:1.0:1775767895.730698:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775767895.730700:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fdc00 x1862022294248576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767920 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767895.730704:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767895.730706:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fdc00 x1862022294248576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767920 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767895.730711:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767895.730712:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775767895.730713:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767895.730715:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:1.0:1775767895.730717:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767895.730718:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299820555 00000100:00000001:1.0:1775767895.730720:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299821885 00000100:00000001:1.0:1775767895.730722:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299820555 00000100:00000001:1.0:1775767895.730724:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299821885 00000100:00000001:1.0:1775767895.730725:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775767895.730726:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767895.730727:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775767895.730729:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767895/real 1775767895] req@ffff9942099fdc00 x1862022294248576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775767895.730734:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775767895.730735:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775767895.730736:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023cf78. 00000400:00000200:1.0:1775767895.730738:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf63b8 00000400:00000010:1.0:1775767895.730739:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf63b8. 00000100:00000001:1.0:1775767895.730740:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775767895.730742:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fdc00 x1862022294248576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775767895.730747:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fdc00 x1862022294248576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767895.730750:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775767895.730751:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767895.730752:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767895.730753:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767895.730755:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fdc00 x1862022294248576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767895.730759:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775767895.730761:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fdc00 x1862022294248576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767920 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767895.730765:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775767895.730766:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767895.730767:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767895.730768:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767895.730769:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775767895.730771:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775767895.730773:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767895.730775:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775767895.730777:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775767895.730779:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fdc00 x1862022294248576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767920 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767895.730788:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294248576:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775767895.730790:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775767895.730792:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fe680 x1862022294248704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767895.730796:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767895.730797:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767895.730798:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775767895.730799:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767895.730800:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767895.730802:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294248704:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775767895.730804:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775767895.730805:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775767895.730805:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767895.730806:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775767895.730808:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543f800. 02000000:00000001:1.0:1775767895.730809:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767895.730811:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775767895.730813:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023cf78. 00000400:00000010:1.0:1775767895.730815:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf63b8. 00000100:00000200:1.0:1775767895.730817:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294248704, portal 4 00000100:00000001:1.0:1775767895.730818:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775767895.730819:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641206400 : -112966068345216 : ffff9942099fe680) 00000100:00000040:1.0:1775767895.730822:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fe680 x1862022294248704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767920 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767895.730826:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775767895.730828:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775767895.730829:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6dd0. 00000100:00000200:1.0:1775767895.730831:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294248704, offset 0 00000400:00000200:1.0:1775767895.730833:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775767895.730836:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767895.730838:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775767895.730840:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775767895.730842:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775767895.730843:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775767895.730845:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775767895.730847:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225116:225116:256:4294967295] 192.168.202.156@tcp LPNI seq info [225112:225112:8:4294967295] 00000400:00000200:1.0:1775767895.730850:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767895.730854:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775767895.730856:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767895.730858:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541800. 00000800:00000200:1.0:1775767895.730860:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775767895.730861:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775767895.730864:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767895.730865:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000100:1.0:1775767895.730868:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767895.730869:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541800. 00000400:00000200:1.0:1775767895.730871:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775767895.730873:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775767895.730875:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6dd0 00000400:00000010:1.0:1775767895.730876:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6dd0. 00000100:00000001:1.0:1775767895.730877:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775767895.730879:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fe680 x1862022294248704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767920 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767895.730884:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767895.730886:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fe680 x1862022294248704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767920 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767895.730890:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767895.730891:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775767895.730893:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767895.730895:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:1.0:1775767895.730896:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767895.730898:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299820578 00000100:00000001:1.0:1775767895.730900:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299821885 00000100:00000001:1.0:1775767895.730915:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299820578 00000100:00000001:1.0:1775767895.730917:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299821885 00000100:00000001:1.0:1775767895.730919:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775767895.730920:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767895.730921:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775767895.730924:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767895/real 1775767895] req@ffff9942099fe680 x1862022294248704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775767895.730929:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775767895.730930:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775767895.730931:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023cf78. 00000400:00000200:1.0:1775767895.730933:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf63b8 00000400:00000010:1.0:1775767895.730934:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf63b8. 00000100:00000001:1.0:1775767895.730935:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775767895.730937:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fe680 x1862022294248704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775767895.730943:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fe680 x1862022294248704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767895.730946:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775767895.730947:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767895.730948:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767895.730949:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767895.730951:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fe680 x1862022294248704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767895.730955:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775767895.730957:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fe680 x1862022294248704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767920 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767895.730962:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775767895.730964:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767895.730965:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767895.730966:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767895.730967:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775767895.730970:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775767895.730972:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767895.730974:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775767895.730975:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775767895.730978:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fe680 x1862022294248704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767920 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767895.730982:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294248704:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775767895.730985:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775767895.730987:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fca80 x1862022294248832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767895.730991:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767895.730992:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767895.730993:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775767895.730994:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767895.730995:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767895.730997:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294248832:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775767895.730999:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775767895.731000:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775767895.731001:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767895.731002:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775767895.731005:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543fc00. 02000000:00000001:1.0:1775767895.731006:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767895.731008:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775767895.731009:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023cf78. 00000400:00000010:1.0:1775767895.731011:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf63b8. 00000100:00000200:1.0:1775767895.731013:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294248832, portal 4 00000100:00000001:1.0:1775767895.731015:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775767895.731016:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641199232 : -112966068352384 : ffff9942099fca80) 00000100:00000040:1.0:1775767895.731020:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fca80 x1862022294248832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767920 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767895.731024:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775767895.731025:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775767895.731027:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6dd0. 00000100:00000200:1.0:1775767895.731028:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294248832, offset 0 00000400:00000200:1.0:1775767895.731030:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775767895.731032:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767895.731034:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775767895.731036:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775767895.731038:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775767895.731039:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775767895.731041:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775767895.731043:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225117:225117:256:4294967295] 192.168.202.156@tcp LPNI seq info [225113:225113:8:4294967295] 00000400:00000200:1.0:1775767895.731046:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767895.731049:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775767895.731052:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767895.731054:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541800. 00000800:00000200:1.0:1775767895.731057:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775767895.731059:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775767895.731061:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767895.731063:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000100:1.0:1775767895.731066:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767895.731067:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541800. 00000400:00000200:1.0:1775767895.731069:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775767895.731071:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775767895.731073:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6dd0 00000400:00000010:1.0:1775767895.731074:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6dd0. 00000100:00000001:1.0:1775767895.731075:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775767895.731077:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fca80 x1862022294248832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767920 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767895.731087:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767895.731089:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fca80 x1862022294248832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767920 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767895.731094:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767895.731095:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775767895.731096:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767895.731098:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:1.0:1775767895.731099:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767895.731101:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299820581 00000100:00000001:1.0:1775767895.731102:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299821885 00000100:00000001:1.0:1775767895.731104:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299820581 00000100:00000001:1.0:1775767895.731105:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299821885 00000100:00000001:1.0:1775767895.731107:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775767895.731108:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767895.731109:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775767895.731111:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767895/real 1775767895] req@ffff9942099fca80 x1862022294248832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775767895.731115:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775767895.731116:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775767895.731117:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023cf78. 00000400:00000200:1.0:1775767895.731118:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf63b8 00000400:00000010:1.0:1775767895.731119:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf63b8. 00000100:00000001:1.0:1775767895.731120:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775767895.731122:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fca80 x1862022294248832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775767895.731127:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fca80 x1862022294248832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767895.731131:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775767895.731133:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767895.731134:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767895.731135:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767895.731137:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fca80 x1862022294248832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767895.731141:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775767895.731143:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fca80 x1862022294248832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767920 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767895.731147:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775767895.731149:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767895.731150:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767895.731150:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767895.731151:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775767895.731169:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775767895.731171:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767895.731173:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775767895.731174:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775767895.731176:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fca80 x1862022294248832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767920 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767895.731181:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294248832:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775767895.731183:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767895.731184:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767895.731186:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fce00 x1862022294248448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767920 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767895.731190:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775767895.731191:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775767895.731192:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543f000. 02000000:00000001:1.0:1775767895.731194:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775767895.731194:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775767895.731196:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:1.0:1775767895.731197:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775767895.731198:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028400. 02000000:00000001:1.0:1775767895.731200:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775767895.731201:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775767895.731202:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fce00. 00000100:00000001:1.0:1775767895.731204:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775767895.731205:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767895.731207:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767895.731209:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fdc00 x1862022294248576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767920 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767895.731213:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775767895.731214:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775767895.731215:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543d400. 02000000:00000001:1.0:1775767895.731216:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775767895.731217:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775767895.731217:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:1.0:1775767895.731218:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775767895.731219:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029800. 02000000:00000001:1.0:1775767895.731220:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775767895.731221:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775767895.731222:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fdc00. 00000100:00000001:1.0:1775767895.731223:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775767895.731224:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767895.731225:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767895.731226:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fe680 x1862022294248704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767920 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767895.731230:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775767895.731231:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775767895.731232:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543f800. 02000000:00000001:1.0:1775767895.731233:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775767895.731233:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775767895.731234:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:1.0:1775767895.731235:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775767895.731236:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002bc00. 02000000:00000001:1.0:1775767895.731237:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775767895.731238:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775767895.731239:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fe680. 00000100:00000001:1.0:1775767895.731241:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775767895.731242:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767895.731243:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767895.731245:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fca80 x1862022294248832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767920 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767895.731248:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775767895.731249:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775767895.731250:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543fc00. 02000000:00000001:1.0:1775767895.731251:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775767895.731252:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775767895.731252:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:1.0:1775767895.731253:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775767895.731254:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002b800. 02000000:00000001:1.0:1775767895.731255:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775767895.731256:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775767895.731256:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fca80. 00000100:00000001:1.0:1775767895.731258:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775767895.731258:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767895.731259:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767895.731261:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767895.731262:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767895.731263:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775767895.731264:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767896.753259:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767896.753260:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767896.753267:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767896.753267:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767896.753267:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767896.753275:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767896.753275:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767896.753276:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767896.753282:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767896.753284:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775767896.753284:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:1.0:1775767896.753292:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767896.753293:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767896.753293:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767896.753302:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767896.753304:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767896.753304:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767896.753306:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767896.753311:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767896.753315:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767896.753318:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767896.753321:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767896.753324:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767896.753327:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767896.753328:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767896.753330:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767896.753335:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767897.073284:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299821885, at 4299821888 00000100:00000010:1.0:1775767897.073299:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cba540. 00000100:00000001:2.0:1775767897.777262:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767897.777262:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767897.777269:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767897.777270:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767897.777271:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767897.777275:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767897.777278:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775767897.777280:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767897.777281:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767897.777284:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767897.777286:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767897.777291:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767897.777292:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767897.777293:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767897.777295:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767897.777302:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00100000:1.0:1775767897.777305:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767897.777306:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767897.777314:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767897.777314:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767897.777314:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767897.777322:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767897.777322:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767897.777337:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767897.777338:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767897.777342:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767897.777346:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767898.801304:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767898.801306:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767898.801311:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767898.801313:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767898.801315:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767898.801320:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767898.801322:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:1.0:1775767898.801328:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767898.801329:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767898.801335:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767898.801336:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767898.801341:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767898.801342:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767898.801348:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767898.801353:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767898.801361:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767898.801365:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767898.801369:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767898.801457:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767898.801459:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767898.801461:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767898.801463:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767898.801465:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767898.801467:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767898.801468:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767898.801469:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767898.801471:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767899.825286:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767899.825287:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767899.825295:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767899.825295:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767899.825298:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767899.825298:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767899.825306:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:1.0:1775767899.825307:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767899.825313:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767899.825313:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767899.825319:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767899.825319:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767899.825323:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767899.825325:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767899.825330:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767899.825331:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767899.825336:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767899.825339:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767900.849273:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767900.849281:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767900.849284:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767900.849291:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767900.849295:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767900.849299:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767900.849301:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767900.849304:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767900.849306:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767900.849327:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767900.849338:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767900.849344:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775767900.849355:0:36:0:(rw.c:1481:ll_writepages()) Process entered 00000100:00100000:3.0:1775767900.849355:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000080:00000001:2.0:1775767900.849362:0:36:0:(file.c:5250:cl_sync_file_range()) Process entered 00000100:00000001:3.0:1775767900.849363:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1775767900.849372:0:36:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00000001:3.0:1775767900.849373:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767900.849391:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000010:2.0:1775767900.849392:0:36:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994210b32630. 00000100:00000001:3.0:1775767900.849396:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767900.849401:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1775767900.849402:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9942060d5800. 00000020:00000010:2.0:1775767900.849410:0:36:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff9942060d5a00. 00000020:00000010:2.0:1775767900.849416:0:36:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942182d6900. 00000008:00000010:2.0:1775767900.849426:0:36:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207ccc2f0. 00020000:00000010:2.0:1775767900.849433:0:36:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994220156e40. 00008000:00000010:2.0:1775767900.849444:0:36:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942077a8e20. 00000080:00000010:2.0:1775767900.849457:0:36:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff9942087472a0. 00000080:00000010:2.0:1775767900.849464:0:36:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b8000. 00000020:00000010:2.0:1775767900.849471:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9942060d4c00. 00000008:00000010:2.0:1775767900.849479:0:36:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942077cd600. 00020000:00000010:2.0:1775767900.849485:0:36:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff99420876d680. 00008000:00000010:2.0:1775767900.849495:0:36:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076cf60. 00000080:00000010:2.0:1775767900.849502:0:36:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff994205066870. 00000020:00000001:2.0:1775767900.849509:0:36:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107759908408 : -112965949643208 : ffff994210b32638) 00000020:00001000:2.0:1775767900.849517:0:36:0:(cl_object.c:831:cl_env_get()) 1@ffff994210b32630 00000020:00000001:2.0:1775767900.849526:0:36:0:(cl_object.c:151:cl_object_top()) ffff99420791a5f8 -> ffff99420791a5f8 00000020:00000001:2.0:1775767900.849533:0:36:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:2.0:1775767900.849537:0:36:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:2.0:1775767900.849542:0:36:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x10176:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:2.0:1775767900.849552:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775767900.849555:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:2.0:1775767900.849560:0:36:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775767900.849569:0:36:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x10176:0x0]io ffff9942077b8060 type 6 ignore/verify layout 1/0 00020000:00000002:2.0:1775767900.849580:0:36:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207dfb6c8) owner 0000000000000000/ffff994201348000 00020000:00000001:2.0:1775767900.849591:0:36:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:2.0:1775767900.849597:0:36:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:2.0:1775767900.849605:0:36:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00020000:00000001:2.0:1775767900.849609:0:36:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767900.849616:0:36:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:2.0:1775767900.849626:0:36:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:2.0:1775767900.849630:0:36:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767900.849638:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775767900.849643:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:2.0:1775767900.849649:0:36:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00020000:00000002:2.0:1775767900.849654:0:36:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207dfb6c8) owner 0000000000000000/ffff994201348000 00000020:00000001:2.0:1775767900.849661:0:36:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767900.849665:0:36:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:2.0:1775767900.849668:0:36:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00020000:00000001:2.0:1775767900.849673:0:36:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00020000:00200000:2.0:1775767900.849679:0:36:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00000002:2.0:1775767900.849686:0:36:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:2.0:1775767900.849692:0:36:0:(lov_io.c:155:lov_sub_get()) Process entered 00020000:00000001:2.0:1775767900.849695:0:36:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000020:00000001:2.0:1775767900.849697:0:36:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775767900.849702:0:36:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994210b32090. 00000020:00000010:2.0:1775767900.849711:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9942060d5200. 00000020:00000010:2.0:1775767900.849717:0:36:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff9942060d4800. 00000020:00000010:2.0:1775767900.849722:0:36:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942182d6c00. 00000008:00000010:2.0:1775767900.849727:0:36:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207cc8b28. 00020000:00000010:2.0:1775767900.849735:0:36:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994220156840. 00008000:00000010:2.0:1775767900.849742:0:36:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942077a9530. 00000080:00000010:2.0:1775767900.849759:0:36:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994208747738. 00000080:00000010:2.0:1775767900.849765:0:36:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077ba910. 00000020:00000010:2.0:1775767900.849773:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994209d2d600. 00000008:00000010:2.0:1775767900.849780:0:36:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942077cd8c0. 00020000:00000010:2.0:1775767900.849785:0:36:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff99420876e400. 00008000:00000010:2.0:1775767900.849789:0:36:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076c3f0. 00000080:00000010:2.0:1775767900.849793:0:36:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942050663c0. 00000020:00000001:2.0:1775767900.849796:0:36:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107759906968 : -112965949644648 : ffff994210b32098) 00000020:00001000:2.0:1775767900.849799:0:36:0:(cl_object.c:831:cl_env_get()) 1@ffff994210b32090 00000020:00000001:2.0:1775767900.849802:0:36:0:(cl_object.c:151:cl_object_top()) ffff99420791ae90 -> ffff99420791a5f8 00000020:00000001:2.0:1775767900.849805:0:36:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:2.0:1775767900.849809:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775767900.849813:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:2.0:1775767900.849817:0:36:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767900.849822:0:36:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767900.849828:0:36:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107621738216 : -112966087813400 : ffff99420876d6e8) 00000020:00000001:2.0:1775767900.849835:0:36:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000100:00000001:1.0:1775767900.849840:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775767900.849844:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767900.849849:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000001:2.0:1775767900.849849:0:36:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767900.849872:0:36:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767900.849873:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fea00 x1862022294248960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1775767900.849879:0:36:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767900.849890:0:36:0:(cl_io.c:314:cl_io_lock()) Process entered 00020000:00000001:2.0:1775767900.849896:0:36:0:(lov_io.c:1108:lov_io_lock()) Process entered 00000100:00000001:1.0:1775767900.849902:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00020000:00000001:2.0:1775767900.849902:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000001:1.0:1775767900.849908:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767900.849908:0:36:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:2.0:1775767900.849912:0:36:0:(cl_io.c:286:cl_lockset_lock()) Process entered 02000000:00000001:1.0:1775767900.849915:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:2.0:1775767900.849916:0:36:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767900.849919:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1775767900.849921:0:36:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767900.849925:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767900.849931:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767900.849936:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294248960:192.168.202.156@tcp:250:kworker.0 00020000:00000001:2.0:1775767900.849937:0:36:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767900.849947:0:36:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000100:00000001:1.0:1775767900.849949:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000001:2.0:1775767900.849951:0:36:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767900.849953:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:2.0:1775767900.849955:0:36:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767900.849957:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767900.849965:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:2.0:1775767900.849965:0:36:0:(cl_io.c:528:cl_io_start()) Process entered 00020000:00000001:2.0:1775767900.849972:0:36:0:(lov_io.c:1114:lov_io_start()) Process entered 02000000:00000010:1.0:1775767900.849973:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543f800. 00020000:00000001:2.0:1775767900.849976:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 02000000:00000001:1.0:1775767900.849980:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767900.849981:0:36:0:(cl_io.c:528:cl_io_start()) Process entered 00000008:00000001:2.0:1775767900.849987:0:36:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000400:00000200:1.0:1775767900.849991:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00000001:2.0:1775767900.849994:0:36:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000400:00000010:1.0:1775767900.849996:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023cf78. 00000400:00000010:1.0:1775767900.850003:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf63b8. 00000008:00000001:2.0:1775767900.850004:0:36:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:2.0:1775767900.850009:0:36:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000100:00000200:1.0:1775767900.850013:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294248960, portal 25 00000008:00000001:2.0:1775767900.850013:0:36:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767900.850023:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000008:00000001:2.0:1775767900.850024:0:36:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000100:00000001:1.0:1775767900.850030:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641207296 : -112966068344320 : ffff9942099fea00) 00000008:00000001:2.0:1775767900.850031:0:36:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000008:00000001:2.0:1775767900.850036:0:36:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107754940544 : -112965954611072 : ffff994210675880) 00000100:00000040:1.0:1775767900.850042:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fea00 x1862022294248960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767925 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000020:2.0:1775767900.850042:0:36:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994210675880 ready 0|-|- wr 1|-|+ rd 0|- 10 in flight 00000008:00000001:2.0:1775767900.850060:0:36:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:2.0:1775767900.850064:0:36:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:2.0:1775767900.850068:0:36:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1775767900.850075:0:36:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000100:00000001:1.0:1775767900.850077:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00000020:2.0:1775767900.850080:0:36:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994218232fc0 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000100:00000040:1.0:1775767900.850089:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775767900.850097:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6dd0. 00000008:00000001:2.0:1775767900.850099:0:36:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:1.0:1775767900.850103:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294248960, offset 0 00000008:00000001:2.0:1775767900.850109:0:36:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767900.850114:0:36:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767900.850121:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00020000:00000001:2.0:1775767900.850122:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767900.850129:0:36:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767900.850139:0:36:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767900.850140:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:2.0:1775767900.850146:0:36:0:(cl_io.c:555:cl_io_end()) Process entered 00000400:00000200:1.0:1775767900.850152:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00020000:00000001:2.0:1775767900.850152:0:36:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00000400:00000200:1.0:1775767900.850187:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775767900.850193:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775767900.850200:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775767900.850205:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00020000:00000001:2.0:1775767900.850207:0:36:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000400:00000200:1.0:1775767900.850214:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225118:225118:256:4294967295] 192.168.202.156@tcp LPNI seq info [225114:225114:8:4294967295] 00000020:00000001:2.0:1775767900.850214:0:36:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000001:2.0:1775767900.850220:0:36:0:(cl_io.c:563:cl_io_end()) Process leaving 00020000:00000001:2.0:1775767900.850226:0:36:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767900.850232:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:2.0:1775767900.850235:0:36:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000020:00000001:2.0:1775767900.850239:0:36:0:(cl_io.c:563:cl_io_end()) Process leaving 00000020:00000001:2.0:1775767900.850245:0:36:0:(cl_io.c:352:cl_io_unlock()) Process entered 00020000:00000001:2.0:1775767900.850250:0:36:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00000400:00000200:1.0:1775767900.850256:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00020000:00000001:2.0:1775767900.850256:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775767900.850260:0:36:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000020:00000001:2.0:1775767900.850264:0:36:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:2.0:1775767900.850269:0:36:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775767900.850277:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00020000:00000001:2.0:1775767900.850277:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1775767900.850285:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541800. 00020000:00000001:2.0:1775767900.850286:0:36:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000020:00000001:2.0:1775767900.850293:0:36:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:2.0:1775767900.850296:0:36:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000800:00000200:1.0:1775767900.850303:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000001:2.0:1775767900.850304:0:36:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000001:2.0:1775767900.850309:0:36:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:2.0:1775767900.850316:0:36:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00000800:00000200:1.1:1775767900.850318:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00020000:00000001:2.0:1775767900.850322:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775767900.850325:0:36:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000001:2.0:1775767900.850331:0:36:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000100:00000001:1.0:1775767900.850336:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767900.850337:0:36:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767900.850347:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767900.850355:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:2.0:1775767900.850359:0:36:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000100:00000001:1.0:1775767900.850362:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299827006 00000020:00000001:2.0:1775767900.850362:0:36:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000020:00000001:2.0:1775767900.850368:0:36:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775767900.850370:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba540. 00000020:00000001:2.0:1775767900.850378:0:36:0:(cl_io.c:71:cl_io_fini()) Process entered 00000100:00000001:1.0:1775767900.850381:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00020000:00000001:2.0:1775767900.850383:0:36:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:2.0:1775767900.850389:0:36:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000100:00000001:1.0:1775767900.850390:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299827006 00000020:00000001:2.0:1775767900.850394:0:36:0:(cl_io.c:71:cl_io_fini()) Process entered 00000100:00000010:1.0:1775767900.850398:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba600. 00000020:00000001:2.0:1775767900.850398:0:36:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:2.0:1775767900.850405:0:36:0:(cl_object.c:925:cl_env_put()) 1@ffff994210b32090 00000100:00000001:1.0:1775767900.850406:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000020:00000010:2.0:1775767900.850415:0:36:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff9942060d4800. 00000100:00000001:1.0:1775767900.850419:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775767900.850424:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767900.850427:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1775767900.850431:0:36:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942182d6c00. 00000100:00000001:1.0:1775767900.850432:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000008:00000010:2.0:1775767900.850442:0:36:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207cc8b28. 00000100:00000001:1.0:1775767900.850444:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767900.850446:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00020000:00000010:2.0:1775767900.850448:0:36:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994220156840. 00000100:00000001:1.0:1775767900.850453:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00008000:00000010:2.0:1775767900.850454:0:36:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942077a9530. 00000100:00000001:1.0:1775767900.850460:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000080:00000010:2.0:1775767900.850462:0:36:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994208747738. 00000100:00000001:1.0:1775767900.850463:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000080:00000010:2.0:1775767900.850471:0:36:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077ba910. 00000100:00000040:1.0:1775767900.850472:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fca80 x1862022294249088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1775767900.850482:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9942060d5200. 00000008:00000010:2.0:1775767900.850490:0:36:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942077cd8c0. 00020000:00000010:2.0:1775767900.850501:0:36:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff99420876e400. 00000100:00000001:1.0:1775767900.850503:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00008000:00000010:2.0:1775767900.850506:0:36:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076c3f0. 00000100:00000001:1.0:1775767900.850507:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:2.0:1775767900.850514:0:36:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942050663c0. 02000000:00000001:1.0:1775767900.850515:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000010:2.0:1775767900.850524:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994209d2d600. 00000020:00000010:2.0:1775767900.850531:0:36:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994210b32090. 02000000:00000001:1.0:1775767900.850532:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00020000:00000001:2.0:1775767900.850535:0:36:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 02000000:00000001:1.0:1775767900.850540:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767900.850540:0:36:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:2.0:1775767900.850544:0:36:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000080:00200000:2.0:1775767900.850549:0:36:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x10176:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:1.0:1775767900.850551:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294249088:192.168.202.156@tcp:38:kworker.0 00000080:00000001:2.0:1775767900.850564:0:36:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000100:00000001:1.0:1775767900.850566:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000001:2.0:1775767900.850567:0:36:0:(cl_io.c:112:cl_io_fini()) Process leaving 02000000:00000001:1.0:1775767900.850574:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000020:00001000:2.0:1775767900.850574:0:36:0:(cl_object.c:925:cl_env_put()) 1@ffff994210b32630 00000020:00000010:2.0:1775767900.850581:0:36:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff9942060d5a00. 02000000:00000001:1.0:1775767900.850582:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1775767900.850586:0:36:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942182d6900. 02000000:00000001:1.0:1775767900.850587:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000010:2.0:1775767900.850594:0:36:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207ccc2f0. 02000000:00000010:1.0:1775767900.850597:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543d400. 00020000:00000010:2.0:1775767900.850602:0:36:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994220156e40. 02000000:00000001:1.0:1775767900.850603:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:2.0:1775767900.850609:0:36:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942077a8e20. 00000080:00000010:2.0:1775767900.850614:0:36:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff9942087472a0. 00000400:00000200:1.0:1775767900.850618:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000080:00000010:2.0:1775767900.850620:0:36:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b8000. 00000400:00000010:1.0:1775767900.850627:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023cc08. 00000020:00000010:2.0:1775767900.850628:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9942060d5800. 00000008:00000010:2.0:1775767900.850641:0:36:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942077cd600. 00000400:00000010:1.0:1775767900.850642:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf67f8. 00020000:00000010:2.0:1775767900.850647:0:36:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff99420876d680. 00000100:00000200:1.0:1775767900.850649:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294249088, portal 10 00000100:00000001:1.0:1775767900.850658:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00008000:00000010:2.0:1775767900.850658:0:36:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076cf60. 00000100:00000001:1.0:1775767900.850665:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641199232 : -112966068352384 : ffff9942099fca80) 00000080:00000010:2.0:1775767900.850665:0:36:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff994205066870. 00000020:00000010:2.0:1775767900.850670:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9942060d4c00. 00000020:00000010:2.0:1775767900.850676:0:36:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994210b32630. 00000080:00000001:2.0:1775767900.850681:0:36:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775767900.850686:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fca80 x1862022294249088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767925 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000001:2.0:1775767900.850688:0:36:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767900.850703:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775767900.850713:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775767900.850718:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6110. 00000100:00000200:1.0:1775767900.850726:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294249088, offset 0 00000400:00000200:1.0:1775767900.850739:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775767900.850747:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767900.850757:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775767900.850763:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775767900.850768:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775767900.850770:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775767900.850773:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775767900.850776:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225119:225119:255:4294967295] 192.168.202.156@tcp LPNI seq info [225115:225115:7:4294967295] 00000400:00000200:1.0:1775767900.850781:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767900.850788:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775767900.850794:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767900.850800:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541e00. 00000800:00000200:1.0:1775767900.850809:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767900.850817:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:1.0:1775767900.850823:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767900.850830:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:1.0:1775767900.850840:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767900.850844:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299825683 00000100:00000001:1.0:1775767900.850850:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299827006 00000100:00000001:1.0:1775767900.850871:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299825683 00000100:00000001:1.0:1775767900.850876:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299827006 00000100:00000001:1.0:1775767900.850882:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775767900.850889:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767900.850897:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767900.850902:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767900.850910:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767900.850913:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1775767900.850917:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775767900.850922:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767900.850926:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775767900.850934:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fe680 x1862022294249216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767900.850951:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767900.850954:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767900.850959:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775767900.850963:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767900.850970:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767900.850974:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294249216:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775767900.850984:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775767900.850988:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775767900.850991:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767900.850995:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775767900.851001:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543f000. 02000000:00000001:1.0:1775767900.851006:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767900.851012:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775767900.851017:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c318. 00000400:00000010:1.0:1775767900.851024:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000100:00000200:1.0:1775767900.851031:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294249216, portal 10 00000100:00000001:1.0:1775767900.851035:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775767900.851037:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641206400 : -112966068345216 : ffff9942099fe680) 00000100:00000040:1.0:1775767900.851056:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fe680 x1862022294249216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767925 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767900.851073:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775767900.851081:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775767900.851087:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6770. 00000100:00000200:1.0:1775767900.851093:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294249216, offset 0 00000400:00000200:1.0:1775767900.851104:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775767900.851112:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767900.851120:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775767900.851125:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775767900.851131:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775767900.851138:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775767900.851144:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775767900.851153:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225120:225120:254:4294967295] 192.168.202.156@tcp LPNI seq info [225116:225116:6:4294967295] 00000400:00000200:1.0:1775767900.851187:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767900.851198:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775767900.851210:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767900.851218:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541300. 00000800:00000200:1.0:1775767900.851227:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767900.851235:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:1.0:1775767900.851241:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767900.851245:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:1.0:1775767900.851252:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767900.851256:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299825676 00000100:00000001:1.0:1775767900.851262:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299827006 00000100:00000001:1.0:1775767900.851267:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299825676 00000100:00000001:1.0:1775767900.851273:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299827006 00000100:00000001:1.0:1775767900.851277:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775767900.851281:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767900.851284:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767900.851288:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767900.851293:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767900.851296:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1775767900.851302:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775767900.851306:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767900.851308:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775767900.851316:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fdc00 x1862022294249344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767900.851334:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767900.851337:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767900.851341:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775767900.851345:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767900.851348:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767900.851354:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294249344:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775767900.851358:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775767900.851360:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775767900.851362:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767900.851364:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775767900.851368:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543c800. 02000000:00000001:1.0:1775767900.851370:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767900.851375:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775767900.851378:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c6e0. 00000400:00000010:1.0:1775767900.851382:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6440. 00000100:00000200:1.0:1775767900.851385:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294249344, portal 4 00000100:00000001:1.0:1775767900.851387:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775767900.851388:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641203712 : -112966068347904 : ffff9942099fdc00) 00000100:00000040:1.0:1775767900.851394:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fdc00 x1862022294249344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767925 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767900.851401:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775767900.851403:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775767900.851405:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6198. 00000100:00000200:1.0:1775767900.851407:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294249344, offset 0 00000400:00000200:1.0:1775767900.851411:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775767900.851416:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767900.851421:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775767900.851424:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775767900.851427:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775767900.851429:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775767900.851432:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775767900.851435:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225121:225121:253:4294967295] 192.168.202.156@tcp LPNI seq info [225117:225117:5:4294967295] 00000400:00000200:1.0:1775767900.851438:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767900.851444:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775767900.851449:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767900.851452:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000800:00000200:1.0:1775767900.851456:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767900.851460:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:1.0:1775767900.851463:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767900.851466:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:1.0:1775767900.851469:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767900.851471:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299825699 00000100:00000001:1.0:1775767900.851474:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299827006 00000100:00000001:1.0:1775767900.851476:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299825699 00000100:00000001:1.0:1775767900.851478:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299827006 00000100:00000001:1.0:1775767900.851482:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775767900.851484:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767900.851486:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767900.851488:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767900.851491:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767900.851492:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1775767900.851494:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775767900.851495:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767900.851496:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775767900.851500:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fce00 x1862022294249472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767900.851507:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767900.851508:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767900.851510:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775767900.851512:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767900.851514:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767900.851517:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294249472:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775767900.851520:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775767900.851521:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775767900.851523:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767900.851525:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775767900.851528:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543c400. 02000000:00000001:1.0:1775767900.851530:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767900.851533:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775767900.851535:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023cdc0. 00000400:00000010:1.0:1775767900.851538:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf62a8. 00000100:00000200:1.0:1775767900.851541:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294249472, portal 4 00000100:00000001:1.0:1775767900.851542:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775767900.851544:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641200128 : -112966068351488 : ffff9942099fce00) 00000100:00000040:1.0:1775767900.851549:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fce00 x1862022294249472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767925 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767900.851556:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775767900.851558:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775767900.851561:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6b28. 00000100:00000200:1.0:1775767900.851564:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294249472, offset 0 00000400:00000200:1.0:1775767900.851568:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775767900.851572:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767900.851575:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775767900.851578:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775767900.851580:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775767900.851582:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775767900.851585:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775767900.851587:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225122:225122:252:4294967295] 192.168.202.156@tcp LPNI seq info [225118:225118:4:4294967295] 00000400:00000200:1.0:1775767900.851591:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767900.851597:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775767900.851601:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767900.851605:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541600. 00000800:00000200:1.0:1775767900.851608:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767900.851612:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:1.0:1775767900.851615:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767900.851618:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:1.0:1775767900.851621:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767900.851623:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299825702 00000100:00000001:1.0:1775767900.851625:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299827006 00000100:00000001:1.0:1775767900.851627:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299825702 00000100:00000001:1.0:1775767900.851629:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299827006 00000100:00000001:1.0:1775767900.851637:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775767900.851641:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767900.851643:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767900.851645:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767900.851648:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767900.851649:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1775767900.851651:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775767900.851652:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767900.851653:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767900.851656:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767901.873253:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767901.873260:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767901.873263:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767901.873276:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767901.873280:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767901.873283:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767901.873292:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767901.873294:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767901.873297:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767901.873319:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767901.873324:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767901.873327:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767901.873331:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767901.873334:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767901.873337:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767901.873339:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767901.873341:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767901.873343:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767902.193226:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299827006, at 4299827008 00000100:00000010:1.0:1775767902.193239:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cba600. 00000100:00000001:1.0:1775767902.897245:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767902.897253:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767902.897257:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767902.897265:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767902.897270:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767902.897274:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767902.897286:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767902.897290:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767902.897294:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767902.897298:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767902.897302:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767902.897303:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767902.897307:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767902.897310:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767902.897313:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767902.897315:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767902.897316:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767902.897318:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767903.921246:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767903.921251:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767903.921254:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767903.921258:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767903.921263:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767903.921265:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767903.921268:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767903.921270:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767903.921272:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767903.921450:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767903.921454:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767903.921456:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767903.921459:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767903.921462:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767903.921464:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767903.921466:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767903.921469:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767903.921472:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:2.0:1775767903.921507:0:2389:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.156:988 00000400:00000100:2.0:1775767903.921571:0:2389:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.156@tcp at host 192.168.202.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:2.0:1775767903.921581:0:2389:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767903.921584:0:2389:0:(router.c:1750:lnet_notify()) 192.168.202.56@tcp notifying 192.168.202.156@tcp: down 00000400:00000200:2.0:1775767903.921589:0:2389:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767903.921592:0:2389:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:2.0:1775767903.921596:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000800:00000010:2.0:1775767903.921599:0:2389:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541800. 00000400:00000200:2.0:1775767903.921604:0:2389:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775767903.921608:0:2389:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775767903.921613:0:2389:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6dd0 00000400:00000010:2.0:1775767903.921615:0:2389:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6dd0. 00000100:00000001:2.0:1775767903.921618:0:2389:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767903.921625:0:2389:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9942099fea00 x1862022294248960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767925 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767903.921650:0:2389:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767903.921655:0:2389:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fea00 x1862022294248960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767925 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767903.921664:0:2389:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767903.921668:0:2389:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1775767903.921680:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000800:00000010:2.0:1775767903.921683:0:2389:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541e00. 00000400:00000200:2.0:1775767903.921687:0:2389:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000100:00000001:1.0:1775767903.921692:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:2.0:1775767903.921692:0:2389:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000100:00000001:1.0:1775767903.921699:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000400:00000200:2.0:1775767903.921701:0:2389:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6110 00000400:00000010:2.0:1775767903.921705:0:2389:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6110. 00000100:00000001:1.0:1775767903.921708:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1775767903.921709:0:2389:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767903.921717:0:2389:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9942099fca80 x1862022294249088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767925 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767903.921726:0:2389:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767903.921738:0:2389:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fca80 x1862022294249088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767925 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767903.921744:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767900/real 1775767903] req@ffff9942099fea00 x1862022294248960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767903.921765:0:2389:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767903.921774:0:2389:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000001:1.0:1775767903.921787:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775767903.921793:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775767903.921798:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023cf78. 00000800:00000100:2.0:1775767903.921803:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000800:00000010:2.0:1775767903.921809:0:2389:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541300. 00000400:00000200:1.0:1775767903.921811:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf63b8 00000400:00000200:2.0:1775767903.921814:0:2389:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000010:1.0:1775767903.921817:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf63b8. 00000100:00000001:1.0:1775767903.921839:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000200:2.0:1775767903.921840:0:2389:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775767903.921849:0:2389:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6770 00000100:00000200:1.0:1775767903.921854:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fea00 x1862022294248960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1775767903.921856:0:2389:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6770. 00000100:00000001:2.0:1775767903.921880:0:2389:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775767903.921883:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fea00 x1862022294248960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767903.921891:0:2389:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9942099fe680 x1862022294249216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767925 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767903.921903:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775767903.921908:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767903.921917:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767903.921922:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767903.921923:0:2389:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767903.921931:0:2389:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fe680 x1862022294249216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767925 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767903.921940:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fea00 x1862022294248960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767903.921976:0:2389:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767903.921981:0:2389:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1775767903.921990:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000800:00000010:2.0:1775767903.921992:0:2389:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000100:00000001:1.0:1775767903.921997:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1775767903.921998:0:2389:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775767903.922013:0:2389:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000100:00000040:1.0:1775767903.922014:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fea00 x1862022294248960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767925 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1775767903.922020:0:2389:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6198 00000400:00000010:2.0:1775767903.922023:0:2389:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6198. 00000100:00000001:2.0:1775767903.922033:0:2389:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1775767903.922048:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000200:2.0:1775767903.922048:0:2389:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9942099fdc00 x1862022294249344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767925 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767903.922059:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767903.922065:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767903.922069:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767903.922078:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775767903.922088:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767903.922088:0:2389:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00080000:1.0:1775767903.922099:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000040:2.0:1775767903.922100:0:2389:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fdc00 x1862022294249344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767925 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767903.922120:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767903.922126:0:2389:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767903.922128:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1775767903.922131:0:2389:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:1.0:1775767903.922144:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fea00 x1862022294248960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767925 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1775767903.922152:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000100:00100000:1.0:1775767903.922207:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294248960:192.168.202.156@tcp:250:kworker.0 00000100:00000001:1.0:1775767903.922215:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767903.922219:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767903.922224:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fea00 x1862022294248960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767925 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767903.922234:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775767903.922239:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000800:00000010:2.0:1775767903.922256:0:2389:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541600. 02000000:00000010:1.0:1775767903.922258:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543f800. 00000400:00000200:2.0:1775767903.922261:0:2389:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 02000000:00000001:1.0:1775767903.922269:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775767903.922276:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000400:00000200:2.0:1775767903.922277:0:2389:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000020:00000040:1.0:1775767903.922280:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000400:00000200:2.0:1775767903.922286:0:2389:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6b28 00000020:00000001:1.0:1775767903.922287:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775767903.922295:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002a800. 00000400:00000010:2.0:1775767903.922295:0:2389:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6b28. 00000100:00000001:2.0:1775767903.922303:0:2389:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:1.0:1775767903.922305:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775767903.922309:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775767903.922314:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fea00. 00000100:00000200:2.0:1775767903.922314:0:2389:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9942099fce00 x1862022294249472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767925 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767903.922324:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775767903.922328:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767903.922336:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767903.922339:0:2389:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1775767903.922340:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775767903.922345:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767903.922348:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000040:2.0:1775767903.922350:0:2389:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fce00 x1862022294249472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767925 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767903.922361:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767900/real 1775767903] req@ffff9942099fce00 x1862022294249472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767925 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767903.922364:0:2389:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767903.922371:0:2389:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000001:1.0:1775767903.922382:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775767903.922384:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775767903.922387:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023cdc0. 00000400:00000200:1.0:1775767903.922393:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf62a8 00000400:00000010:1.0:1775767903.922395:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf62a8. 00000100:00000001:1.0:1775767903.922398:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775767903.922402:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fce00 x1862022294249472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775767903.922414:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fce00 x1862022294249472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767903.922427:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775767903.922428:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767903.922431:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767903.922432:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767903.922436:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fce00 x1862022294249472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767903.922449:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775767903.922453:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fce00 x1862022294249472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767925 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767903.922461:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775767903.922464:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767903.922466:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767903.922468:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767903.922470:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775767903.922476:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775767903.922479:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767903.922482:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775767903.922485:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775767903.922489:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fce00 x1862022294249472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767925 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767903.922499:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294249472:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775767903.922505:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775767903.922510:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767900/real 1775767903] req@ffff9942099fdc00 x1862022294249344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775767903.922519:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775767903.922521:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775767903.922523:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c6e0. 00000400:00000200:1.0:1775767903.922526:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6440 00000400:00000010:1.0:1775767903.922528:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6440. 00000100:00000001:1.0:1775767903.922531:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775767903.922534:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fdc00 x1862022294249344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775767903.922543:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fdc00 x1862022294249344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767903.922550:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775767903.922551:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767903.922553:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767903.922555:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767903.922558:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fdc00 x1862022294249344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767903.922566:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775767903.922570:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fdc00 x1862022294249344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767925 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767903.922578:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775767903.922580:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767903.922582:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767903.922584:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767903.922586:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775767903.922590:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775767903.922593:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767903.922597:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775767903.922600:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775767903.922604:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fdc00 x1862022294249344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767925 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767903.922613:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294249344:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775767903.922618:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775767903.922623:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767900/real 1775767903] req@ffff9942099fe680 x1862022294249216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775767903.922631:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775767903.922633:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775767903.922635:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c318. 00000400:00000200:1.0:1775767903.922638:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000400:00000010:1.0:1775767903.922640:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000100:00000001:1.0:1775767903.922642:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775767903.922645:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fe680 x1862022294249216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775767903.922654:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fe680 x1862022294249216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767903.922660:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775767903.922662:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767903.922664:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767903.922665:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767903.922668:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fe680 x1862022294249216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767903.922674:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775767903.922678:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fe680 x1862022294249216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767925 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767903.922685:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775767903.922687:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767903.922689:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767903.922691:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767903.922693:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775767903.922697:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775767903.922700:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767903.922705:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775767903.922709:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775767903.922714:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fe680 x1862022294249216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767925 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767903.922723:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294249216:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775767903.922728:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775767903.922733:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767900/real 1775767903] req@ffff9942099fca80 x1862022294249088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775767903.922742:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775767903.922744:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775767903.922747:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023cc08. 00000400:00000200:1.0:1775767903.922750:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf67f8 00000400:00000010:1.0:1775767903.922752:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf67f8. 00000100:00000001:1.0:1775767903.922755:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775767903.922758:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fca80 x1862022294249088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775767903.922767:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fca80 x1862022294249088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767903.922772:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775767903.922773:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767903.922775:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767903.922777:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767903.922781:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fca80 x1862022294249088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767903.922787:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775767903.922792:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fca80 x1862022294249088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767925 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767903.922804:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775767903.922807:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767903.922809:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767903.922810:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767903.922812:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775767903.922816:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775767903.922819:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767903.922838:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775767903.922841:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775767903.922846:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fca80 x1862022294249088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767925 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767903.922855:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294249088:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775767903.922860:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767903.922862:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767903.922866:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fce00 x1862022294249472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767925 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767903.922873:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775767903.922875:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775767903.922877:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543c400. 02000000:00000001:1.0:1775767903.922880:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775767903.922882:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775767903.922883:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:1.0:1775767903.922885:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775767903.922886:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029800. 02000000:00000001:1.0:1775767903.922889:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775767903.922890:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775767903.922891:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fce00. 00000100:00000001:1.0:1775767903.922894:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775767903.922895:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767903.922898:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767903.922902:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fdc00 x1862022294249344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767925 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767903.922909:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775767903.922911:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775767903.922913:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543c800. 02000000:00000001:1.0:1775767903.922922:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775767903.922924:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775767903.922925:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:1.0:1775767903.922928:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775767903.922930:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002bc00. 02000000:00000001:1.0:1775767903.922934:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775767903.922935:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775767903.922937:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fdc00. 00000100:00000001:1.0:1775767903.922940:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775767903.922942:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767903.922944:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767903.922948:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fe680 x1862022294249216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767925 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767903.922955:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775767903.922956:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775767903.922958:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543f000. 02000000:00000001:1.0:1775767903.922962:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775767903.922963:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775767903.922965:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:1.0:1775767903.922967:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775767903.922969:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002b800. 02000000:00000001:1.0:1775767903.922972:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775767903.922973:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775767903.922975:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fe680. 00000100:00000001:1.0:1775767903.922977:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775767903.922978:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767903.922980:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767903.922984:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fca80 x1862022294249088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767925 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767903.922995:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775767903.922997:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775767903.922998:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543d400. 02000000:00000001:1.0:1775767903.923001:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775767903.923002:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775767903.923003:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:1.0:1775767903.923006:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775767903.923007:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028000. 02000000:00000001:1.0:1775767903.923010:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775767903.923011:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775767903.923012:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fca80. 00000100:00000001:1.0:1775767903.923014:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775767903.923015:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767903.923017:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767903.923027:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767903.923030:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767903.923032:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775767903.923034:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767904.945226:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767904.945235:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767904.945238:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767904.945243:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767904.945247:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767904.945252:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767904.945254:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767904.945256:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767904.945259:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767904.945308:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767904.945312:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767904.945314:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767904.945318:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767904.945324:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767904.945328:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767904.945333:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767904.945336:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767904.945345:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767905.969316:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767905.969327:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767905.969330:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767905.969339:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775767905.969339:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767905.969349:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767905.969350:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767905.969358:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767905.969361:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767905.969368:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00100000:2.0:1775767905.969369:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767905.969369:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767905.969373:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767905.969377:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767905.969378:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767905.969379:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767905.969385:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767905.969386:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767905.969398:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767905.969403:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767905.969406:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767905.969409:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767905.969412:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767905.969415:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767905.969416:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767905.969421:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767905.969426:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767905.969458:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767905.969466:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775767905.969470:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775767905.969475:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775767905.969480:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775767905.969490:0:13471:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5158 00000400:00000200:1.0:1775767905.969505:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000080:00000001:2.0:1775767905.969505:0:36:0:(rw.c:1481:ll_writepages()) Process entered 00000080:00000001:2.0:1775767905.969512:0:36:0:(file.c:5250:cl_sync_file_range()) Process entered 00000020:00000001:2.0:1775767905.969517:0:36:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000400:00000200:1.0:1775767905.969518:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775767905.969524:0:13471:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775767905.969529:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000010:2.0:1775767905.969529:0:36:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994210b32630. 00000100:00000001:1.0:1775767905.969539:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767905.969544:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767905.969547:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000010:2.0:1775767905.969548:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994209d2d600. 00000020:00000001:1.0:1775767905.969556:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775767905.969560:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000010:2.0:1775767905.969561:0:36:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994209d2cc00. 00000020:00000001:1.0:1775767905.969567:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000010:2.0:1775767905.969569:0:36:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942182d6900. 00000020:00000040:1.0:1775767905.969572:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:1.0:1775767905.969577:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000008:00000010:2.0:1775767905.969581:0:36:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207ccc2f0. 00020000:00000010:2.0:1775767905.969588:0:36:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994220156e40. 00008000:00000010:2.0:1775767905.969594:0:36:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942077a8e20. 00000020:00000001:1.0:1775767905.969597:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000080:00000010:2.0:1775767905.969602:0:36:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff9942087472a0. 00000100:00000040:1.0:1775767905.969606:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000080:00000010:2.0:1775767905.969610:0:36:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b8000. 00000100:00000001:1.0:1775767905.969611:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767905.969617:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000010:2.0:1775767905.969618:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994209d2dc00. 00000100:00000001:1.0:1775767905.969624:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775767905.969631:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000008:00000010:2.0:1775767905.969634:0:36:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942077cd600. 00000100:00080000:1.0:1775767905.969637:0:13471:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00020000:00000010:2.0:1775767905.969643:0:36:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff99420876d680. 00000100:00000001:1.0:1775767905.969644:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767905.969653:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00008000:00000010:2.0:1775767905.969654:0:36:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076cf60. 02000000:00000001:1.0:1775767905.969659:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000010:2.0:1775767905.969662:0:36:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff994205066870. 02000000:00000001:1.0:1775767905.969665:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767905.969671:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000020:00000001:2.0:1775767905.969672:0:36:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107759908408 : -112965949643208 : ffff994210b32638) 00000100:00000001:1.0:1775767905.969676:0:13471:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:2.0:1775767905.969679:0:36:0:(cl_object.c:831:cl_env_get()) 1@ffff994210b32630 00000020:00000001:2.0:1775767905.969685:0:36:0:(cl_object.c:151:cl_object_top()) ffff9942078a76c0 -> ffff9942078a76c0 00000100:00000010:1.0:1775767905.969687:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207889880. 00000020:00000001:2.0:1775767905.969690:0:36:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000040:1.0:1775767905.969693:0:13471:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 00000080:00000001:2.0:1775767905.969694:0:36:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:2.0:1775767905.969700:0:36:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 02000000:00000001:1.0:1775767905.969702:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000020:00000001:2.0:1775767905.969707:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775767905.969709:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:2.0:1775767905.969711:0:36:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775767905.969716:0:36:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017a:0x0]io ffff9942077b8060 type 6 ignore/verify layout 1/0 02000000:00000001:1.0:1775767905.969719:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775767905.969723:0:36:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207c1f9b0) owner 0000000000000000/ffff994201348000 00000100:00000001:1.0:1775767905.969726:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00020000:00000001:2.0:1775767905.969729:0:36:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 02000000:00000001:1.0:1775767905.969732:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00020000:00000001:2.0:1775767905.969734:0:36:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:2.0:1775767905.969738:0:36:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 02000000:00000001:1.0:1775767905.969740:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767905.969741:0:36:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767905.969748:0:36:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:2.0:1775767905.969752:0:36:0:(lov_io.c:203:lov_io_subio_init()) Process entered 02000000:00000010:1.0:1775767905.969755:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543f000. 00020000:00000001:2.0:1775767905.969756:0:36:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767905.969764:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775767905.969766:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00000001:1.0:1775767905.969771:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767905.969775:0:36:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00020000:00000002:2.0:1775767905.969779:0:36:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207c1f9b0) owner 0000000000000000/ffff994201348000 00000020:00000001:2.0:1775767905.969784:0:36:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767905.969791:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207889880 x1862022294249600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000001:2.0:1775767905.969792:0:36:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:2.0:1775767905.969796:0:36:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00020000:00000001:2.0:1775767905.969801:0:36:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00020000:00200000:2.0:1775767905.969820:0:36:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00000002:2.0:1775767905.969830:0:36:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00000100:00000001:1.0:1775767905.969834:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00020000:00000001:2.0:1775767905.969835:0:36:0:(lov_io.c:155:lov_sub_get()) Process entered 00020000:00000001:2.0:1775767905.969839:0:36:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000020:00000001:2.0:1775767905.969841:0:36:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00000001:1.0:1775767905.969845:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1775767905.969848:0:36:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994210b32090. 00000020:00000010:2.0:1775767905.969855:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994209d2de00. 00000100:00000040:1.0:1775767905.969856:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f11856b8] to pc [ptlrpcd_rcv+-1] req@ffff994207889880 x1862022294249600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1775767905.969863:0:36:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994209d2d000. 00000020:00000010:2.0:1775767905.969874:0:36:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942182d6c00. 00000008:00000010:2.0:1775767905.969897:0:36:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207cc8b28. 00020000:00000010:2.0:1775767905.969901:0:36:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994220156840. 00000100:00000001:1.0:1775767905.969905:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:2.0:1775767905.969908:0:36:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942077a9530. 00000100:00080000:1.0:1775767905.969911:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000080:00000010:2.0:1775767905.969916:0:36:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994208747738. 00000100:00080000:1.0:1775767905.969919:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775767905.969923:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000080:00000010:2.0:1775767905.969924:0:36:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077ba910. 00000100:00080000:1.0:1775767905.969926:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775767905.969935:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000020:00000010:2.0:1775767905.969938:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9942060d4c00. 00000100:00080000:1.0:1775767905.969939:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5158 00000400:00000200:1.0:1775767905.969950:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767905.969955:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000008:00000010:2.0:1775767905.969956:0:36:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942077cd8c0. 00000100:00080000:1.0:1775767905.969960:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00020000:00000010:2.0:1775767905.969963:0:36:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff99420876e400. 00000100:00000040:1.0:1775767905.969967:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00008000:00000010:2.0:1775767905.969970:0:36:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076c3f0. 00000100:00000001:1.0:1775767905.969974:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000080:00000010:2.0:1775767905.969979:0:36:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942050663c0. 00000100:00000040:1.0:1775767905.969982:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767905.969989:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775767905.969989:0:36:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107759906968 : -112965949644648 : ffff994210b32098) 00000020:00000001:1.0:1775767905.970002:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00001000:2.0:1775767905.970003:0:36:0:(cl_object.c:831:cl_env_get()) 1@ffff994210b32090 00000020:00000040:1.0:1775767905.970007:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:1.0:1775767905.970014:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000001:2.0:1775767905.970016:0:36:0:(cl_object.c:151:cl_object_top()) ffff9942078a7788 -> ffff9942078a76c0 00000020:00000040:1.0:1775767905.970019:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:2.0:1775767905.970029:0:36:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:1.0:1775767905.970030:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:2.0:1775767905.970039:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:1.0:1775767905.970040:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:2.0:1775767905.970046:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:2.0:1775767905.970051:0:36:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767905.970052:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767905.970061:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00020000:00000001:2.0:1775767905.970063:0:36:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767905.970068:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00020000:00000001:2.0:1775767905.970075:0:36:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107621738216 : -112966087813400 : ffff99420876d6e8) 00000100:00000001:1.0:1775767905.970077:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775767905.970088:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000020:00000001:2.0:1775767905.970088:0:36:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000020:00000001:2.0:1775767905.970098:0:36:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767905.970101:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00020000:00000001:2.0:1775767905.970107:0:36:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767905.970108:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767905.970118:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000020:00000001:2.0:1775767905.970118:0:36:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767905.970128:0:36:0:(cl_io.c:314:cl_io_lock()) Process entered 00020000:00000001:2.0:1775767905.970132:0:36:0:(lov_io.c:1108:lov_io_lock()) Process entered 02000000:00000001:1.0:1775767905.970133:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767905.970141:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767905.970142:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775767905.970148:0:36:0:(cl_io.c:314:cl_io_lock()) Process entered 00000100:00000001:1.0:1775767905.970149:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775767905.970193:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000020:00000001:2.0:1775767905.970194:0:36:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:2.0:1775767905.970199:0:36:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1775767905.970201:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767905.970206:0:36:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767905.970208:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767905.970216:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775767905.970220:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207888e00. 00020000:00000001:2.0:1775767905.970221:0:36:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1775767905.970231:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:2.0:1775767905.970232:0:36:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:2.0:1775767905.970239:0:36:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767905.970241:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775767905.970247:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767905.970250:0:36:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767905.970253:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775767905.970261:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000020:00000001:2.0:1775767905.970262:0:36:0:(cl_io.c:528:cl_io_start()) Process entered 02000000:00000001:1.0:1775767905.970266:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767905.970270:0:36:0:(lov_io.c:1114:lov_io_start()) Process entered 02000000:00000010:1.0:1775767905.970276:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543c800. 00020000:00000001:2.0:1775767905.970277:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775767905.970284:0:36:0:(cl_io.c:528:cl_io_start()) Process entered 00000100:00000001:1.0:1775767905.970287:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1775767905.970292:0:36:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000100:00100000:1.0:1775767905.970301:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207888e00 x1862022294249728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000008:00000001:2.0:1775767905.970302:0:36:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000008:00000001:2.0:1775767905.970314:0:36:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000100:00000001:1.0:1775767905.970322:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000008:00000020:2.0:1775767905.970322:0:36:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000100:00000001:1.0:1775767905.970327:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1775767905.970330:0:36:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775767905.970339:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001812f966] to pc [ptlrpcd_rcv+-1] req@ffff994207888e00 x1862022294249728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:2.0:1775767905.970341:0:36:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000008:00000001:2.0:1775767905.970346:0:36:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000008:00000001:2.0:1775767905.970355:0:36:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107884691392 : -112965824860224 : ffff994218232fc0) 00000008:00000020:2.0:1775767905.970368:0:36:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994218232fc0 ready 0|-|- wr 1|-|+ rd 0|- 10 in flight 00000008:00000001:2.0:1775767905.970380:0:36:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000100:00000001:1.0:1775767905.970384:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000008:00000020:2.0:1775767905.970384:0:36:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:2.0:1775767905.970389:0:36:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1775767905.970392:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000008:00000001:2.0:1775767905.970399:0:36:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000100:00080000:1.0:1775767905.970401:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775767905.970406:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000008:00000020:2.0:1775767905.970407:0:36:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994210675880 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000100:00080000:1.0:1775767905.970409:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775767905.970420:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775767905.970430:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5158 00000008:00000001:2.0:1775767905.970433:0:36:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1775767905.970444:0:36:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767905.970448:0:36:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767905.970450:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767905.970452:0:36:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767905.970454:0:36:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767905.970456:0:36:0:(cl_io.c:555:cl_io_end()) Process entered 00020000:00000001:2.0:1775767905.970458:0:36:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00020000:00000001:2.0:1775767905.970460:0:36:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000020:00000001:2.0:1775767905.970461:0:36:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000001:2.0:1775767905.970462:0:36:0:(cl_io.c:563:cl_io_end()) Process leaving 00020000:00000001:2.0:1775767905.970464:0:36:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767905.970466:0:36:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000020:00000001:2.0:1775767905.970468:0:36:0:(cl_io.c:563:cl_io_end()) Process leaving 00000020:00000001:2.0:1775767905.970470:0:36:0:(cl_io.c:352:cl_io_unlock()) Process entered 00020000:00000001:2.0:1775767905.970474:0:36:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00020000:00000001:2.0:1775767905.970476:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775767905.970478:0:36:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000020:00000001:2.0:1775767905.970481:0:36:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:2.0:1775767905.970483:0:36:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767905.970485:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767905.970488:0:36:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000020:00000001:2.0:1775767905.970491:0:36:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:2.0:1775767905.970493:0:36:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000020:00000001:2.0:1775767905.970494:0:36:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000001:2.0:1775767905.970496:0:36:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:2.0:1775767905.970498:0:36:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00020000:00000001:2.0:1775767905.970499:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775767905.970501:0:36:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000001:2.0:1775767905.970502:0:36:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00020000:00000001:2.0:1775767905.970504:0:36:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767905.970505:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767905.970508:0:36:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000020:00000001:2.0:1775767905.970509:0:36:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000020:00000001:2.0:1775767905.970510:0:36:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767905.970512:0:36:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:2.0:1775767905.970514:0:36:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:2.0:1775767905.970516:0:36:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:2.0:1775767905.970517:0:36:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:2.0:1775767905.970519:0:36:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:2.0:1775767905.970521:0:36:0:(cl_object.c:925:cl_env_put()) 1@ffff994210b32090 00000020:00000010:2.0:1775767905.970526:0:36:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994209d2d000. 00000020:00000010:2.0:1775767905.970534:0:36:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942182d6c00. 00000008:00000010:2.0:1775767905.970539:0:36:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207cc8b28. 00020000:00000010:2.0:1775767905.970543:0:36:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994220156840. 00008000:00000010:2.0:1775767905.970547:0:36:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942077a9530. 00000080:00000010:2.0:1775767905.970551:0:36:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994208747738. 00000080:00000010:2.0:1775767905.970557:0:36:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077ba910. 00000020:00000010:2.0:1775767905.970561:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994209d2de00. 00000008:00000010:2.0:1775767905.970568:0:36:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942077cd8c0. 00020000:00000010:2.0:1775767905.970572:0:36:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff99420876e400. 00008000:00000010:2.0:1775767905.970578:0:36:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076c3f0. 00000080:00000010:2.0:1775767905.970582:0:36:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942050663c0. 00000020:00000010:2.0:1775767905.970585:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9942060d4c00. 00000020:00000010:2.0:1775767905.970589:0:36:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994210b32090. 00020000:00000001:2.0:1775767905.970595:0:36:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:2.0:1775767905.970598:0:36:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:2.0:1775767905.970601:0:36:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000080:00200000:2.0:1775767905.970604:0:36:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:2.0:1775767905.970609:0:36:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:2.0:1775767905.970610:0:36:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:2.0:1775767905.970614:0:36:0:(cl_object.c:925:cl_env_put()) 1@ffff994210b32630 00000020:00000010:2.0:1775767905.970618:0:36:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994209d2cc00. 00000020:00000010:2.0:1775767905.970623:0:36:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942182d6900. 00000008:00000010:2.0:1775767905.970626:0:36:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207ccc2f0. 00020000:00000010:2.0:1775767905.970629:0:36:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994220156e40. 00008000:00000010:2.0:1775767905.970632:0:36:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942077a8e20. 00000080:00000010:2.0:1775767905.970634:0:36:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff9942087472a0. 00000080:00000010:2.0:1775767905.970637:0:36:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b8000. 00000020:00000010:2.0:1775767905.970639:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994209d2d600. 00000008:00000010:2.0:1775767905.970641:0:36:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942077cd600. 00020000:00000010:2.0:1775767905.970644:0:36:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff99420876d680. 00008000:00000010:2.0:1775767905.970647:0:36:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076cf60. 00000080:00000010:2.0:1775767905.970650:0:36:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff994205066870. 00000020:00000010:2.0:1775767905.970652:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994209d2dc00. 00000020:00000010:2.0:1775767905.970654:0:36:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994210b32630. 00000080:00000001:2.0:1775767905.970656:0:36:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:2.0:1775767905.970659:0:36:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775767905.970667:0:36:0:(rw.c:1481:ll_writepages()) Process entered 00000080:00000001:2.0:1775767905.970669:0:36:0:(file.c:5250:cl_sync_file_range()) Process entered 00000020:00000001:2.0:1775767905.970670:0:36:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775767905.970675:0:36:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994210b32630. 00000020:00000010:2.0:1775767905.970678:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9942060d4c00. 00000020:00000010:2.0:1775767905.970682:0:36:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff9942060d5800. 00000020:00000010:2.0:1775767905.970685:0:36:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942182d6900. 00000008:00000010:2.0:1775767905.970688:0:36:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207ccc2f0. 00020000:00000010:2.0:1775767905.970691:0:36:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994220156e40. 00008000:00000010:2.0:1775767905.970695:0:36:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942077a8e20. 00000080:00000010:2.0:1775767905.970698:0:36:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff9942087472a0. 00000080:00000010:2.0:1775767905.970701:0:36:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b8000. 00000020:00000010:2.0:1775767905.970704:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9942060d5a00. 00000008:00000010:2.0:1775767905.970707:0:36:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942077cd600. 00020000:00000010:2.0:1775767905.970710:0:36:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff99420876d680. 00008000:00000010:2.0:1775767905.970713:0:36:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076cf60. 00000080:00000010:2.0:1775767905.970716:0:36:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff994205066870. 00000020:00000001:2.0:1775767905.970718:0:36:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107759908408 : -112965949643208 : ffff994210b32638) 00000020:00001000:2.0:1775767905.970721:0:36:0:(cl_object.c:831:cl_env_get()) 1@ffff994210b32630 00000020:00000001:2.0:1775767905.970724:0:36:0:(cl_object.c:151:cl_object_top()) ffff9942078a7f58 -> ffff9942078a7f58 00000020:00000001:2.0:1775767905.970726:0:36:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:2.0:1775767905.970728:0:36:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:2.0:1775767905.970730:0:36:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017b:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:2.0:1775767905.970732:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775767905.970733:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:2.0:1775767905.970735:0:36:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775767905.970737:0:36:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017b:0x0]io ffff9942077b8060 type 6 ignore/verify layout 1/0 00020000:00000002:2.0:1775767905.970741:0:36:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207c1f000) owner 0000000000000000/ffff994201348000 00020000:00000001:2.0:1775767905.970744:0:36:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:2.0:1775767905.970745:0:36:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:2.0:1775767905.970747:0:36:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00020000:00000001:2.0:1775767905.970748:0:36:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767905.970750:0:36:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:2.0:1775767905.970753:0:36:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:2.0:1775767905.970754:0:36:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767905.970756:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775767905.970757:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:2.0:1775767905.970758:0:36:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00020000:00000002:2.0:1775767905.970760:0:36:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207c1f000) owner 0000000000000000/ffff994201348000 00000020:00000001:2.0:1775767905.970762:0:36:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767905.970764:0:36:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:2.0:1775767905.970765:0:36:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00020000:00000001:2.0:1775767905.970767:0:36:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00020000:00200000:2.0:1775767905.970769:0:36:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00000002:2.0:1775767905.970771:0:36:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:2.0:1775767905.970774:0:36:0:(lov_io.c:155:lov_sub_get()) Process entered 00020000:00000001:2.0:1775767905.970776:0:36:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000020:00000001:2.0:1775767905.970777:0:36:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775767905.970779:0:36:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994210b32090. 00000020:00000010:2.0:1775767905.970782:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9942060d5200. 00000020:00000010:2.0:1775767905.970785:0:36:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff9942060d4800. 00000020:00000010:2.0:1775767905.970787:0:36:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942182d6c00. 00000008:00000010:2.0:1775767905.970790:0:36:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207cc8b28. 00020000:00000010:2.0:1775767905.970793:0:36:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994220156840. 00008000:00000010:2.0:1775767905.970796:0:36:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942077a9530. 00000080:00000010:2.0:1775767905.970798:0:36:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994208747738. 00000080:00000010:2.0:1775767905.970800:0:36:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077ba910. 00000020:00000010:2.0:1775767905.970819:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994209d2dc00. 00000008:00000010:2.0:1775767905.970823:0:36:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942077cd8c0. 00020000:00000010:2.0:1775767905.970826:0:36:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff99420876e400. 00008000:00000010:2.0:1775767905.970828:0:36:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076c3f0. 00000080:00000010:2.0:1775767905.970831:0:36:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942050663c0. 00000020:00000001:2.0:1775767905.970833:0:36:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107759906968 : -112965949644648 : ffff994210b32098) 00000020:00001000:2.0:1775767905.970835:0:36:0:(cl_object.c:831:cl_env_get()) 1@ffff994210b32090 00000020:00000001:2.0:1775767905.970838:0:36:0:(cl_object.c:151:cl_object_top()) ffff9942078a75f8 -> ffff9942078a7f58 00000020:00000001:2.0:1775767905.970840:0:36:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:2.0:1775767905.970842:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775767905.970843:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:2.0:1775767905.970845:0:36:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767905.970847:0:36:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767905.970849:0:36:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107621738216 : -112966087813400 : ffff99420876d6e8) 00000020:00000001:2.0:1775767905.970851:0:36:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000020:00000001:2.0:1775767905.970852:0:36:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767905.970854:0:36:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767905.970856:0:36:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767905.970858:0:36:0:(cl_io.c:314:cl_io_lock()) Process entered 00020000:00000001:2.0:1775767905.970860:0:36:0:(lov_io.c:1108:lov_io_lock()) Process entered 00020000:00000001:2.0:1775767905.970861:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775767905.970862:0:36:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:2.0:1775767905.970863:0:36:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:2.0:1775767905.970865:0:36:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767905.970866:0:36:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767905.970868:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767905.970869:0:36:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767905.970871:0:36:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:2.0:1775767905.970872:0:36:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767905.970873:0:36:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767905.970875:0:36:0:(cl_io.c:528:cl_io_start()) Process entered 00020000:00000001:2.0:1775767905.970876:0:36:0:(lov_io.c:1114:lov_io_start()) Process entered 00020000:00000001:2.0:1775767905.970878:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775767905.970879:0:36:0:(cl_io.c:528:cl_io_start()) Process entered 00000008:00000001:2.0:1775767905.970880:0:36:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000008:00000001:2.0:1775767905.970882:0:36:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000008:00000001:2.0:1775767905.970885:0:36:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:2.0:1775767905.970887:0:36:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:2.0:1775767905.970888:0:36:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1775767905.970890:0:36:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000008:00000001:2.0:1775767905.970891:0:36:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000008:00000001:2.0:1775767905.970893:0:36:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107884688256 : -112965824863360 : ffff994218232380) 00000008:00000020:2.0:1775767905.970895:0:36:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994218232380 ready 0|-|- wr 1|-|+ rd 0|- 9 in flight 00000008:00000001:2.0:1775767905.970898:0:36:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:2.0:1775767905.970899:0:36:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:2.0:1775767905.970901:0:36:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1775767905.970903:0:36:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000008:00000020:2.0:1775767905.970905:0:36:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994210674fc0 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000008:00000001:2.0:1775767905.970908:0:36:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1775767905.970910:0:36:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767905.970912:0:36:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767905.970914:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767905.970915:0:36:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767905.970917:0:36:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767905.970918:0:36:0:(cl_io.c:555:cl_io_end()) Process entered 00020000:00000001:2.0:1775767905.970920:0:36:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00020000:00000001:2.0:1775767905.970921:0:36:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000020:00000001:2.0:1775767905.970922:0:36:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000001:2.0:1775767905.970923:0:36:0:(cl_io.c:563:cl_io_end()) Process leaving 00020000:00000001:2.0:1775767905.970925:0:36:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767905.970926:0:36:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000020:00000001:2.0:1775767905.970928:0:36:0:(cl_io.c:563:cl_io_end()) Process leaving 00000020:00000001:2.0:1775767905.970929:0:36:0:(cl_io.c:352:cl_io_unlock()) Process entered 00020000:00000001:2.0:1775767905.970930:0:36:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00020000:00000001:2.0:1775767905.970932:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775767905.970933:0:36:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000020:00000001:2.0:1775767905.970934:0:36:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:2.0:1775767905.970935:0:36:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767905.970937:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767905.970938:0:36:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000020:00000001:2.0:1775767905.970941:0:36:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:2.0:1775767905.970944:0:36:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000020:00000001:2.0:1775767905.970947:0:36:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000001:2.0:1775767905.970951:0:36:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:2.0:1775767905.970953:0:36:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00020000:00000001:2.0:1775767905.970955:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775767905.970959:0:36:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000001:2.0:1775767905.970963:0:36:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00020000:00000001:2.0:1775767905.970966:0:36:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767905.970968:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767905.970971:0:36:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000020:00000001:2.0:1775767905.970973:0:36:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000020:00000001:2.0:1775767905.970976:0:36:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767905.970981:0:36:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:2.0:1775767905.970987:0:36:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:2.0:1775767905.970991:0:36:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:2.0:1775767905.970995:0:36:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:2.0:1775767905.971000:0:36:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:2.0:1775767905.971004:0:36:0:(cl_object.c:925:cl_env_put()) 1@ffff994210b32090 00000020:00000010:2.0:1775767905.971009:0:36:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff9942060d4800. 00000020:00000010:2.0:1775767905.971014:0:36:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942182d6c00. 00000008:00000010:2.0:1775767905.971022:0:36:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207cc8b28. 00020000:00000010:2.0:1775767905.971028:0:36:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994220156840. 00008000:00000010:2.0:1775767905.971033:0:36:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942077a9530. 00000080:00000010:2.0:1775767905.971041:0:36:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994208747738. 00000080:00000010:2.0:1775767905.971045:0:36:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077ba910. 00000020:00000010:2.0:1775767905.971052:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9942060d5200. 00000008:00000010:2.0:1775767905.971064:0:36:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942077cd8c0. 00020000:00000010:2.0:1775767905.971073:0:36:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff99420876e400. 00008000:00000010:2.0:1775767905.971080:0:36:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076c3f0. 00000800:00000200:0.1:1775767905.971082:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000080:00000010:2.0:1775767905.971087:0:36:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942050663c0. 00000800:00000200:0.1:1775767905.971088:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000020:00000010:2.0:1775767905.971095:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994209d2dc00. 00000020:00000010:2.0:1775767905.971105:0:36:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994210b32090. 00020000:00000001:2.0:1775767905.971114:0:36:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:2.0:1775767905.971120:0:36:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:2.0:1775767905.971124:0:36:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000080:00200000:2.0:1775767905.971128:0:36:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:2.0:1775767905.971134:0:36:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:2.0:1775767905.971136:0:36:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:2.0:1775767905.971142:0:36:0:(cl_object.c:925:cl_env_put()) 1@ffff994210b32630 00000020:00000010:2.0:1775767905.971149:0:36:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff9942060d5800. 00000020:00000010:2.0:1775767905.971154:0:36:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942182d6900. 00000008:00000010:2.0:1775767905.971188:0:36:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207ccc2f0. 00020000:00000010:2.0:1775767905.971191:0:36:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994220156e40. 00008000:00000010:2.0:1775767905.971194:0:36:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942077a8e20. 00000080:00000010:2.0:1775767905.971196:0:36:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff9942087472a0. 00000080:00000010:2.0:1775767905.971200:0:36:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b8000. 00000020:00000010:2.0:1775767905.971202:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9942060d4c00. 00000008:00000010:2.0:1775767905.971205:0:36:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942077cd600. 00020000:00000010:2.0:1775767905.971208:0:36:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff99420876d680. 00008000:00000010:2.0:1775767905.971210:0:36:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076cf60. 00000080:00000010:2.0:1775767905.971212:0:36:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff994205066870. 00000020:00000010:2.0:1775767905.971214:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9942060d5a00. 00000020:00000010:2.0:1775767905.971216:0:36:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994210b32630. 00000080:00000001:2.0:1775767905.971218:0:36:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:2.0:1775767905.971220:0:36:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775767905.971224:0:36:0:(rw.c:1481:ll_writepages()) Process entered 00000080:00000001:2.0:1775767905.971226:0:36:0:(file.c:5250:cl_sync_file_range()) Process entered 00000020:00000001:2.0:1775767905.971227:0:36:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775767905.971230:0:36:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994210b32630. 00000020:00000010:2.0:1775767905.971234:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994209d2dc00. 00000020:00000010:2.0:1775767905.971237:0:36:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994209d2d600. 00000020:00000010:2.0:1775767905.971240:0:36:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942182d6900. 00000008:00000010:2.0:1775767905.971244:0:36:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207ccc2f0. 00020000:00000010:2.0:1775767905.971248:0:36:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994220156e40. 00008000:00000010:2.0:1775767905.971251:0:36:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942077a8e20. 00000080:00000010:2.0:1775767905.971254:0:36:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff9942087472a0. 00000080:00000010:2.0:1775767905.971257:0:36:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b8000. 00000020:00000010:2.0:1775767905.971260:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994209d2cc00. 00000008:00000010:2.0:1775767905.971262:0:36:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942077cd600. 00020000:00000010:2.0:1775767905.971264:0:36:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff99420876d680. 00008000:00000010:2.0:1775767905.971267:0:36:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076cf60. 00000080:00000010:2.0:1775767905.971269:0:36:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff994205066870. 00000020:00000001:2.0:1775767905.971271:0:36:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107759908408 : -112965949643208 : ffff994210b32638) 00000020:00001000:2.0:1775767905.971274:0:36:0:(cl_object.c:831:cl_env_get()) 1@ffff994210b32630 00000020:00000001:2.0:1775767905.971282:0:36:0:(cl_object.c:151:cl_object_top()) ffff99420791a850 -> ffff99420791a850 00000020:00000001:2.0:1775767905.971284:0:36:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:2.0:1775767905.971285:0:36:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:2.0:1775767905.971287:0:36:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017f:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:2.0:1775767905.971290:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775767905.971291:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:2.0:1775767905.971293:0:36:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775767905.971295:0:36:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017f:0x0]io ffff9942077b8060 type 6 ignore/verify layout 1/0 00020000:00000002:2.0:1775767905.971298:0:36:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207dfbd90) owner 0000000000000000/ffff994201348000 00020000:00000001:2.0:1775767905.971300:0:36:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:2.0:1775767905.971301:0:36:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:2.0:1775767905.971303:0:36:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00020000:00000001:2.0:1775767905.971304:0:36:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767905.971306:0:36:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:2.0:1775767905.971307:0:36:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:2.0:1775767905.971309:0:36:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767905.971310:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775767905.971311:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:2.0:1775767905.971312:0:36:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00020000:00000002:2.0:1775767905.971313:0:36:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207dfbd90) owner 0000000000000000/ffff994201348000 00000020:00000001:2.0:1775767905.971316:0:36:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767905.971317:0:36:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:2.0:1775767905.971319:0:36:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00020000:00000001:2.0:1775767905.971320:0:36:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00020000:00200000:2.0:1775767905.971322:0:36:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00000002:2.0:1775767905.971324:0:36:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:2.0:1775767905.971327:0:36:0:(lov_io.c:155:lov_sub_get()) Process entered 00020000:00000001:2.0:1775767905.971328:0:36:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000020:00000001:2.0:1775767905.971329:0:36:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775767905.971333:0:36:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994210b32090. 00000020:00000010:2.0:1775767905.971336:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994209d2de00. 00000020:00000010:2.0:1775767905.971339:0:36:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994209d2d000. 00000020:00000010:2.0:1775767905.971342:0:36:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942182d6c00. 00000008:00000010:2.0:1775767905.971345:0:36:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207cc8b28. 00020000:00000010:2.0:1775767905.971347:0:36:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994220156840. 00008000:00000010:2.0:1775767905.971350:0:36:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942077a9530. 00000080:00000010:2.0:1775767905.971353:0:36:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994208747738. 00000080:00000010:2.0:1775767905.971355:0:36:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077ba910. 00000020:00000010:2.0:1775767905.971358:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9942060d5a00. 00000008:00000010:2.0:1775767905.971361:0:36:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942077cd8c0. 00020000:00000010:2.0:1775767905.971363:0:36:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff99420876e400. 00008000:00000010:2.0:1775767905.971366:0:36:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076c3f0. 00000080:00000010:2.0:1775767905.971369:0:36:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942050663c0. 00000020:00000001:2.0:1775767905.971371:0:36:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107759906968 : -112965949644648 : ffff994210b32098) 00000020:00001000:2.0:1775767905.971373:0:36:0:(cl_object.c:831:cl_env_get()) 1@ffff994210b32090 00000020:00000001:2.0:1775767905.971375:0:36:0:(cl_object.c:151:cl_object_top()) ffff99420791a210 -> ffff99420791a850 00000020:00000001:2.0:1775767905.971376:0:36:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:2.0:1775767905.971378:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775767905.971379:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:2.0:1775767905.971380:0:36:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767905.971382:0:36:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767905.971384:0:36:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107621738216 : -112966087813400 : ffff99420876d6e8) 00000020:00000001:2.0:1775767905.971386:0:36:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000020:00000001:2.0:1775767905.971388:0:36:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767905.971390:0:36:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767905.971392:0:36:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767905.971393:0:36:0:(cl_io.c:314:cl_io_lock()) Process entered 00020000:00000001:2.0:1775767905.971394:0:36:0:(lov_io.c:1108:lov_io_lock()) Process entered 00020000:00000001:2.0:1775767905.971396:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775767905.971396:0:36:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:2.0:1775767905.971397:0:36:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:2.0:1775767905.971399:0:36:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767905.971402:0:36:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767905.971403:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767905.971405:0:36:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767905.971406:0:36:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:2.0:1775767905.971408:0:36:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767905.971409:0:36:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767905.971411:0:36:0:(cl_io.c:528:cl_io_start()) Process entered 00020000:00000001:2.0:1775767905.971412:0:36:0:(lov_io.c:1114:lov_io_start()) Process entered 00020000:00000001:2.0:1775767905.971413:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775767905.971414:0:36:0:(cl_io.c:528:cl_io_start()) Process entered 00000008:00000001:2.0:1775767905.971415:0:36:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000008:00000001:2.0:1775767905.971417:0:36:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000008:00000001:2.0:1775767905.971419:0:36:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:2.0:1775767905.971420:0:36:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:2.0:1775767905.971421:0:36:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1775767905.971423:0:36:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000008:00000001:2.0:1775767905.971425:0:36:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000008:00000001:2.0:1775767905.971426:0:36:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107754938304 : -112965954613312 : ffff994210674fc0) 00000008:00000020:2.0:1775767905.971429:0:36:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994210674fc0 ready 0|-|- wr 1|-|+ rd 0|- 9 in flight 00000008:00000001:2.0:1775767905.971433:0:36:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:2.0:1775767905.971434:0:36:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:2.0:1775767905.971435:0:36:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1775767905.971437:0:36:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000008:00000020:2.0:1775767905.971439:0:36:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994218232380 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000008:00000001:2.0:1775767905.971442:0:36:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1775767905.971444:0:36:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767905.971446:0:36:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767905.971449:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767905.971450:0:36:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767905.971452:0:36:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767905.971453:0:36:0:(cl_io.c:555:cl_io_end()) Process entered 00020000:00000001:2.0:1775767905.971455:0:36:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00020000:00000001:2.0:1775767905.971456:0:36:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000020:00000001:2.0:1775767905.971457:0:36:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000001:2.0:1775767905.971459:0:36:0:(cl_io.c:563:cl_io_end()) Process leaving 00020000:00000001:2.0:1775767905.971460:0:36:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767905.971462:0:36:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000020:00000001:2.0:1775767905.971463:0:36:0:(cl_io.c:563:cl_io_end()) Process leaving 00000020:00000001:2.0:1775767905.971464:0:36:0:(cl_io.c:352:cl_io_unlock()) Process entered 00020000:00000001:2.0:1775767905.971466:0:36:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00020000:00000001:2.0:1775767905.971467:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775767905.971469:0:36:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000020:00000001:2.0:1775767905.971470:0:36:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:2.0:1775767905.971471:0:36:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767905.971473:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767905.971474:0:36:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000020:00000001:2.0:1775767905.971475:0:36:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:2.0:1775767905.971477:0:36:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000020:00000001:2.0:1775767905.971478:0:36:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000001:2.0:1775767905.971479:0:36:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:2.0:1775767905.971480:0:36:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00020000:00000001:2.0:1775767905.971481:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775767905.971483:0:36:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000001:2.0:1775767905.971484:0:36:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00020000:00000001:2.0:1775767905.971485:0:36:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767905.971487:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767905.971488:0:36:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000020:00000001:2.0:1775767905.971490:0:36:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000020:00000001:2.0:1775767905.971491:0:36:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767905.971493:0:36:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:2.0:1775767905.971494:0:36:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:2.0:1775767905.971495:0:36:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:2.0:1775767905.971496:0:36:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:2.0:1775767905.971497:0:36:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:2.0:1775767905.971499:0:36:0:(cl_object.c:925:cl_env_put()) 1@ffff994210b32090 00000020:00000010:2.0:1775767905.971502:0:36:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994209d2d000. 00000020:00000010:2.0:1775767905.971506:0:36:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942182d6c00. 00000008:00000010:2.0:1775767905.971510:0:36:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207cc8b28. 00020000:00000010:2.0:1775767905.971513:0:36:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994220156840. 00008000:00000010:2.0:1775767905.971516:0:36:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942077a9530. 00000080:00000010:2.0:1775767905.971519:0:36:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994208747738. 00000080:00000010:2.0:1775767905.971521:0:36:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077ba910. 00000020:00000010:2.0:1775767905.971524:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994209d2de00. 00000008:00000010:2.0:1775767905.971526:0:36:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942077cd8c0. 00020000:00000010:2.0:1775767905.971529:0:36:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff99420876e400. 00008000:00000010:2.0:1775767905.971531:0:36:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076c3f0. 00000080:00000010:2.0:1775767905.971534:0:36:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942050663c0. 00000020:00000010:2.0:1775767905.971536:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9942060d5a00. 00000020:00000010:2.0:1775767905.971539:0:36:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994210b32090. 00020000:00000001:2.0:1775767905.971541:0:36:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:2.0:1775767905.971543:0:36:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:2.0:1775767905.971545:0:36:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000080:00200000:2.0:1775767905.971546:0:36:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017f:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:2.0:1775767905.971549:0:36:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:2.0:1775767905.971551:0:36:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:2.0:1775767905.971552:0:36:0:(cl_object.c:925:cl_env_put()) 1@ffff994210b32630 00000020:00000010:2.0:1775767905.971554:0:36:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994209d2d600. 00000020:00000010:2.0:1775767905.971557:0:36:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942182d6900. 00000008:00000010:2.0:1775767905.971560:0:36:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207ccc2f0. 00020000:00000010:2.0:1775767905.971562:0:36:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994220156e40. 00008000:00000010:2.0:1775767905.971565:0:36:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942077a8e20. 00000080:00000010:2.0:1775767905.971567:0:36:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff9942087472a0. 00000080:00000010:2.0:1775767905.971570:0:36:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b8000. 00000020:00000010:2.0:1775767905.971572:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994209d2dc00. 00000008:00000010:2.0:1775767905.971575:0:36:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942077cd600. 00020000:00000010:2.0:1775767905.971577:0:36:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff99420876d680. 00008000:00000010:2.0:1775767905.971579:0:36:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076cf60. 00000080:00000010:2.0:1775767905.971582:0:36:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff994205066870. 00000020:00000010:2.0:1775767905.971584:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994209d2cc00. 00000020:00000010:2.0:1775767905.971586:0:36:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994210b32630. 00000080:00000001:2.0:1775767905.971589:0:36:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:2.0:1775767905.971591:0:36:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767905.973546:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767905.973558:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775767905.973564:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775767905.973572:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767905.973577:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767905.973594:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767905.973598:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775767905.973605:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775767905.973609:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:1.0:1775767905.973613:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775767905.973619:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:1.0:1775767905.973626:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:1.0:1775767905.973632:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:1.0:1775767905.973639:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767905.973642:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767905.973646:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767905.973650:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775767905.973656:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:1.0:1775767905.973663:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775767905.973668:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767905.973674:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775767905.973683:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767905.973689:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767905.973695:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775767905.973701:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775767905.973710:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767905.973713:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775767905.973721:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff99420788b100. 00000020:00000040:1.0:1775767905.973729:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:1.0:1775767905.973737:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775767905.973740:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767905.973746:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775767905.973749:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775767905.973755:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775767905.973765:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543c400. 00000100:00000001:1.0:1775767905.973776:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767905.973789:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff99420788b100 x1862022294249856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775767905.973828:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775767905.973835:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767905.973847:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e26cf977] to pc [ptlrpcd_rcv+-1] req@ffff99420788b100 x1862022294249856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767905.973880:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767905.973889:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767905.973899:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775767905.973904:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775767905.973908:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775767905.973916:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775767905.973923:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5158 00000400:00000200:1.0:1775767905.973935:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767905.973957:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775767905.973965:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775767905.973973:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767905.973978:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767905.973983:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767905.973987:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775767905.973993:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775767905.973996:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:1.0:1775767905.974000:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775767905.974003:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:1.0:1775767905.974005:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:1.0:1775767905.974008:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:1.0:1775767905.974014:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767905.974018:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767905.974022:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767905.974028:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775767905.974033:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:1.0:1775767905.974042:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775767905.974048:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767905.974055:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775767905.974059:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767905.974063:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767905.974068:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775767905.974072:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:1.0:1775767905.974080:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767905.974083:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775767905.974089:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207889f80. 00000020:00000040:1.0:1775767905.974094:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:1.0:1775767905.974099:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775767905.974102:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767905.974104:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775767905.974108:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775767905.974111:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775767905.974119:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543f800. 00000100:00000001:1.0:1775767905.974125:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767905.974133:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207889f80 x1862022294249984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775767905.974146:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775767905.974151:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767905.974181:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ed2274f4] to pc [ptlrpcd_rcv+-1] req@ffff994207889f80 x1862022294249984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767905.974206:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767905.974214:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767905.974221:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775767905.974225:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775767905.974229:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775767905.974235:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775767905.974240:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5158 00000400:00000200:1.0:1775767905.974297:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767905.974306:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775767905.974313:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775767905.974317:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767905.974322:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767905.974328:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767905.974332:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775767905.974337:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000800:00000200:0.1:1775767905.974338:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000040:1.0:1775767905.974340:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:1.0:1775767905.974345:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000800:00000200:0.1:1775767905.974345:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000020:00000040:1.0:1775767905.974349:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:1.0:1775767905.974355:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:1.0:1775767905.974359:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:1.0:1775767905.974367:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767905.974371:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767905.974378:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767905.974382:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775767905.974388:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:1.0:1775767905.974395:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775767905.974401:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767905.974406:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775767905.974411:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767905.974415:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767905.974419:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775767905.974422:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:1.0:1775767905.974429:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767905.974434:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775767905.974441:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207888700. 00000020:00000040:1.0:1775767905.974446:0:13471:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:1.0:1775767905.974452:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775767905.974455:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767905.974458:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775767905.974462:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775767905.974467:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775767905.974474:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543e800. 00000100:00000001:1.0:1775767905.974480:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767905.974490:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207888700 x1862022294250112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775767905.974508:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775767905.974511:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767905.974519:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009f2ba301] to pc [ptlrpcd_rcv+-1] req@ffff994207888700 x1862022294250112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767905.974535:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767905.974543:0:13471:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5168) 00000800:00000200:0.1:1775767905.974753:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767905.974756:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775767905.975076:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767905.975079:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775767905.975358:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767905.975359:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000100:00000001:1.0:1775767906.993282:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767906.993293:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767906.993296:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767906.993306:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767906.993312:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767906.993317:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767906.993319:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767906.993322:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767906.993330:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767906.993345:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767906.993350:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767906.993352:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767906.993357:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767906.993363:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767906.993369:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767906.993373:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767906.993376:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767906.993379:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767906.993471:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767906.993475:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767906.993477:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767906.993480:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767906.993483:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767906.993486:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767906.993488:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767906.993490:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767906.993492:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767907.377325:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299832129, at 4299832192 00000100:00000010:1.0:1775767907.377340:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3900. 00000100:00000001:2.0:1775767908.017289:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767908.017290:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767908.017290:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767908.017303:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767908.017304:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767908.017305:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767908.017306:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767908.017313:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767908.017314:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767908.017315:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775767908.017321:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767908.017326:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1775767908.017327:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775767908.017328:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767908.017331:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767908.017335:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767908.017337:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767908.017338:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767908.017342:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767908.017343:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767908.017350:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767908.017351:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767908.017354:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767908.017356:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767908.017357:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767908.017361:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767908.017364:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767909.041318:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767909.041323:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767909.041327:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767909.041330:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767909.041333:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767909.041334:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767909.041349:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:1.0:1775767909.041350:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767909.041363:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767909.041364:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767909.041374:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767909.041376:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767909.041389:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767909.041393:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767909.041397:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767909.041398:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767909.041401:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767909.041411:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767910.065266:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767910.065273:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767910.065275:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767910.065282:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767910.065284:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767910.065291:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767910.065292:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767910.065294:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767910.065296:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767910.065304:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767910.065312:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767910.065312:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767910.065319:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767910.065320:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767910.065324:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767910.065324:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767910.065328:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767910.065331:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767911.089270:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767911.089276:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767911.089278:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767911.089284:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:3.0:1775767911.089284:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767911.089294:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767911.089294:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767911.089300:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767911.089302:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1775767911.089309:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1775767911.089310:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00100000:1.0:1775767911.089311:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767911.089317:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767911.089319:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767911.089322:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767911.089324:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767911.089330:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767911.089332:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1775767911.089334:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775767911.089336:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767911.089338:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775767911.089344:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775767911.089354:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775767911.089366:0:13471:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5163 00000400:00000200:2.0:1775767911.089381:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767911.089392:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775767911.089397:0:13471:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775767911.089404:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767911.089409:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767911.089413:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767911.089417:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775767911.089423:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775767911.089426:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:2.0:1775767911.089430:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775767911.089434:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:2.0:1775767911.089437:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:2.0:1775767911.089440:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:2.0:1775767911.089444:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767911.089447:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767911.089449:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767911.089451:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775767911.089455:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:2.0:1775767911.089459:0:13471:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775767911.089462:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767911.089465:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775767911.089471:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767911.089474:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767911.089477:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775767911.089480:0:13471:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775767911.089493:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfce00. 00000020:00000040:2.0:1775767911.089497:0:13471:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:2.0:1775767911.089502:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775767911.089504:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767911.089508:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775767911.089511:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775767911.089517:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775767911.089528:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1d400. 00000100:00000001:2.0:1775767911.089536:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767911.089548:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfce00 x1862022294250240/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775767911.089561:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775767911.089566:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767911.089573:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e5a7debe] to pc [ptlrpcd_rcv+-1] req@ffff994207dfce00 x1862022294250240/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767911.089599:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775767911.089602:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767911.089606:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775767911.089607:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775767911.089609:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775767911.089612:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775767911.089616:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5163 00000400:00000200:2.0:1775767911.089621:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767911.089634:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775767911.089637:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775767911.089643:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767911.089648:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767911.089653:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767911.089656:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775767911.089660:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775767911.089663:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:2.0:1775767911.089667:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775767911.089670:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:2.0:1775767911.089676:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:2.0:1775767911.089680:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:2.0:1775767911.089688:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767911.089691:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767911.089695:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767911.089701:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775767911.089707:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00080000:2.0:1775767911.089714:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775767911.089717:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767911.089722:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775767911.089730:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767911.089745:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767911.089749:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775767911.089766:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775767911.089772:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767911.089775:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775767911.089781:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dff480. 00000020:00000040:2.0:1775767911.089786:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:2.0:1775767911.089791:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775767911.089794:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767911.089798:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775767911.089801:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775767911.089804:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775767911.089811:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1c800. 00000100:00000001:2.0:1775767911.089816:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767911.089825:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dff480 x1862022294250368/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775767911.089834:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775767911.089838:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767911.089841:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093a15639] to pc [ptlrpcd_rcv+-1] req@ffff994207dff480 x1862022294250368/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767911.089852:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775767911.089855:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767911.089858:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775767911.089859:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775767911.089862:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775767911.089868:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775767911.089871:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5163 00000400:00000200:2.0:1775767911.089876:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767911.089879:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775767911.089885:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775767911.089888:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767911.089891:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767911.089895:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767911.089897:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775767911.089899:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775767911.089901:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:2.0:1775767911.089904:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775767911.089906:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:2.0:1775767911.089908:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:2.0:1775767911.089912:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:2.0:1775767911.089917:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767911.089919:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767911.089931:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767911.089935:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775767911.089937:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:2.0:1775767911.089942:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775767911.089946:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767911.089948:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775767911.089950:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767911.089952:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767911.089954:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775767911.089956:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775767911.089960:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767911.089962:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775767911.089968:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfed80. 00000020:00000040:2.0:1775767911.089971:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:2.0:1775767911.089974:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775767911.089975:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767911.089979:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775767911.089981:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775767911.089984:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775767911.089988:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1ec00. 00000100:00000001:2.0:1775767911.089992:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767911.089996:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfed80 x1862022294250496/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775767911.090008:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775767911.090010:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767911.090015:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb9b9eae] to pc [ptlrpcd_rcv+-1] req@ffff994207dfed80 x1862022294250496/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.1:1775767911.090025:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:2.0:1775767911.090026:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775767911.090029:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:0.1:1775767911.090032:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000100:00080000:2.0:1775767911.090033:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1775767911.090035:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775767911.090037:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775767911.090040:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775767911.090045:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5163 00000400:00000200:2.0:1775767911.090051:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767911.090054:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775767911.090057:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775767911.090059:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767911.090060:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767911.090062:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767911.090063:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775767911.090065:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775767911.090067:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:2.0:1775767911.090069:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775767911.090072:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:2.0:1775767911.090074:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:2.0:1775767911.090077:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:2.0:1775767911.090082:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767911.090085:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767911.090086:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767911.090088:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775767911.090089:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:2.0:1775767911.090091:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775767911.090095:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767911.090097:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775767911.090099:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767911.090102:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767911.090103:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775767911.090105:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:2.0:1775767911.090109:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767911.090110:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775767911.090113:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfe300. 00000020:00000040:2.0:1775767911.090116:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:2.0:1775767911.090121:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775767911.090123:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767911.090127:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775767911.090129:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775767911.090132:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775767911.090136:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1f000. 00000100:00000001:2.0:1775767911.090140:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767911.090148:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfe300 x1862022294250624/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775767911.090186:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775767911.090189:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767911.090196:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c661fef] to pc [ptlrpcd_rcv+-1] req@ffff994207dfe300 x1862022294250624/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767911.090207:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775767911.090212:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767911.090215:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1775767911.090216:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775767911.090217:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775767911.090219:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775767911.090224:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5163 00000400:00000200:2.0:1775767911.090227:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767911.090232:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775767911.090237:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775767911.090245:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767911.090249:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767911.090255:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767911.090259:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775767911.090266:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775767911.090269:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:2.0:1775767911.090272:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775767911.090275:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:2.0:1775767911.090280:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:2.0:1775767911.090287:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:2.0:1775767911.090293:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767911.090298:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767911.090302:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767911.090306:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775767911.090311:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:2.0:1775767911.090317:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775767911.090323:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767911.090327:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775767911.090331:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767911.090336:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767911.090341:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775767911.090344:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:2.0:1775767911.090350:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767911.090355:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775767911.090362:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfc700. 00000020:00000040:2.0:1775767911.090366:0:13471:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:2.0:1775767911.090372:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775767911.090374:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767911.090377:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775767911.090379:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775767911.090382:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775767911.090387:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1e800. 00000100:00000001:2.0:1775767911.090391:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767911.090400:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfc700 x1862022294250752/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775767911.090418:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775767911.090422:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767911.090428:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a176dd1] to pc [ptlrpcd_rcv+-1] req@ffff994207dfc700 x1862022294250752/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767911.090445:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767911.090451:0:13471:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5173) 00000800:00000200:0.1:1775767911.090605:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767911.090607:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775767911.090868:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767911.090870:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775767911.091108:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767911.091111:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775767911.091357:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767911.091359:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000100:00000001:1.0:1775767912.113252:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767912.113261:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767912.113263:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767912.113273:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767912.113279:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767912.113282:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767912.113284:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767912.113290:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767912.113293:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767912.113294:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767912.113297:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767912.113298:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767912.113307:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767912.113322:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767912.113327:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767912.113331:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767912.113334:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767912.113338:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767912.433318:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299837246, at 4299837248 00000100:00000010:1.0:1775767912.433342:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a38a0. 00000100:00000001:3.0:1775767913.137303:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767913.137304:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767913.137315:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767913.137316:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767913.137321:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767913.137323:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767913.137335:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775767913.137336:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767913.137347:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767913.137352:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767913.137355:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767913.137358:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767913.137358:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767913.137364:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767913.137366:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767913.137372:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767913.137374:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767913.137377:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767914.161312:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767914.161321:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767914.161328:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767914.161329:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767914.161336:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:3.0:1775767914.161339:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767914.161340:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767914.161349:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1775767914.161351:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767914.161353:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767914.161362:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767914.161363:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767914.161366:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767914.161367:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767914.161379:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767914.161388:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767914.161392:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767914.161398:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767915.185290:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767915.185302:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767915.185302:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767915.185306:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767915.185310:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767915.185315:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767915.185316:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:1.0:1775767915.185323:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767915.185323:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767915.185332:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767915.185334:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767915.185339:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767915.185343:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767915.185343:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767915.185355:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767915.185364:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767915.185368:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767915.185373:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767916.209273:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767916.209276:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767916.209281:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767916.209283:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767916.209284:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767916.209287:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767916.209292:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775767916.209293:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767916.209299:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767916.209299:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767916.209306:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767916.209306:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775767916.209309:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775767916.209310:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767916.209314:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775767916.209316:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775767916.209319:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767916.209324:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767916.209328:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767916.209331:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767916.209433:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767916.209437:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767916.209439:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767916.209442:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767916.209444:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767916.209447:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767916.209449:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767916.209451:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767916.209454:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767916.209474:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767916.209479:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767916.209483:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767916.209490:0:13471:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5168 00000400:00000200:3.0:1775767916.209501:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767916.209508:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767916.209511:0:13471:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767916.209514:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767916.209518:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767916.209520:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767916.209522:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767916.209525:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767916.209527:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:3.0:1775767916.209529:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767916.209531:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:3.0:1775767916.209533:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:3.0:1775767916.209535:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:3.0:1775767916.209537:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767916.209538:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767916.209539:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767916.209541:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767916.209544:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:3.0:1775767916.209546:0:13471:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767916.209548:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767916.209550:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767916.209553:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767916.209556:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767916.209558:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775767916.209561:0:13471:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767916.209573:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8e00. 00000020:00000040:3.0:1775767916.209578:0:13471:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:3.0:1775767916.209582:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767916.209584:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767916.209588:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767916.209592:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767916.209599:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767916.209611:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b1000. 00000100:00000001:3.0:1775767916.209620:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767916.209631:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8e00 x1862022294250880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767916.209644:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767916.209649:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767916.209653:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000864a6866] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8e00 x1862022294250880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767916.209670:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767916.209673:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767916.209676:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775767916.209677:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767916.209679:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767916.209681:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767916.209686:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5168 00000400:00000200:3.0:1775767916.209690:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767916.209692:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767916.209694:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767916.209696:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767916.209698:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767916.209700:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767916.209701:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767916.209703:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767916.209731:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:3.0:1775767916.209734:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767916.209736:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:3.0:1775767916.209739:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:3.0:1775767916.209743:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:3.0:1775767916.209750:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767916.209754:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767916.209759:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767916.209765:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767916.209772:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00080000:3.0:1775767916.209777:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767916.209781:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767916.209784:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767916.209791:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767916.209794:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767916.209796:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767916.209802:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775767916.209807:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767916.209809:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767916.209815:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ab480. 00000020:00000040:3.0:1775767916.209818:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:3.0:1775767916.209822:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767916.209823:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767916.209825:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767916.209826:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767916.209829:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767916.209833:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b3400. 00000100:00000001:3.0:1775767916.209838:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767916.209844:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ab480 x1862022294251008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767916.209856:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767916.209858:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767916.209864:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000067a0ba74] to pc [ptlrpcd_rcv+-1] req@ffff9942099ab480 x1862022294251008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767916.209879:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767916.209883:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767916.209899:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775767916.209901:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767916.209903:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767916.209908:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767916.209915:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5168 00000400:00000200:3.0:1775767916.209925:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767916.209932:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767916.209935:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767916.209942:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767916.209946:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767916.209949:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767916.209951:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767916.209957:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767916.209958:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:3.0:1775767916.209959:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767916.209961:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:3.0:1775767916.209965:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:3.0:1775767916.209969:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:3.0:1775767916.209972:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767916.209973:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767916.209974:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767916.209976:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767916.209981:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:3.0:1775767916.209984:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767916.209986:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767916.209990:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767916.209995:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767916.209998:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767916.210001:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767916.210003:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775767916.210015:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767916.210018:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767916.210025:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a9180. 00000020:00000040:3.0:1775767916.210029:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:3.0:1775767916.210032:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767916.210034:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767916.210039:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767916.210042:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767916.210056:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767916.210061:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2c00. 00000100:00000001:3.0:1775767916.210066:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767916.210072:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a9180 x1862022294251136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767916.210081:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767916.210086:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767916.210091:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bdc2586] to pc [ptlrpcd_rcv+-1] req@ffff9942099a9180 x1862022294251136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.1:1775767916.210098:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:3.0:1775767916.210104:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775767916.210104:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000100:00080000:3.0:1775767916.210108:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767916.210115:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775767916.210118:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767916.210120:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767916.210123:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767916.210126:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5168 00000400:00000200:3.0:1775767916.210132:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767916.210139:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767916.210142:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767916.210144:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767916.210148:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767916.210150:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767916.210153:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767916.210182:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767916.210185:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:3.0:1775767916.210189:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767916.210194:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:3.0:1775767916.210198:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:3.0:1775767916.210203:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:3.0:1775767916.210213:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767916.210219:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767916.210224:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767916.210227:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767916.210231:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:3.0:1775767916.210236:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767916.210239:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767916.210243:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767916.210247:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767916.210251:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767916.210253:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767916.210255:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:3.0:1775767916.210257:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767916.210258:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767916.210265:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8700. 00000020:00000040:3.0:1775767916.210267:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:3.0:1775767916.210272:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767916.210273:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767916.210278:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767916.210280:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767916.210283:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767916.210289:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0400. 00000100:00000001:3.0:1775767916.210295:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767916.210302:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8700 x1862022294251264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767916.210313:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767916.210316:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767916.210324:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000faa25e3b] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8700 x1862022294251264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767916.210343:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767916.210348:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767916.210356:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775767916.210359:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767916.210361:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767916.210362:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767916.210365:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5168 00000400:00000200:3.0:1775767916.210368:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767916.210373:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767916.210376:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767916.210380:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767916.210385:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767916.210389:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767916.210392:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767916.210394:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767916.210395:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:3.0:1775767916.210404:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767916.210407:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:3.0:1775767916.210409:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:3.0:1775767916.210412:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:3.0:1775767916.210419:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767916.210421:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767916.210426:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767916.210430:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767916.210433:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:3.0:1775767916.210437:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767916.210442:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767916.210445:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767916.210448:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767916.210453:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767916.210455:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767916.210459:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:3.0:1775767916.210465:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767916.210467:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767916.210471:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8380. 00000020:00000040:3.0:1775767916.210474:0:13471:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:3.0:1775767916.210479:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767916.210480:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767916.210483:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767916.210486:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767916.210489:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767916.210493:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2000. 00000100:00000001:3.0:1775767916.210498:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767916.210506:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8380 x1862022294251392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767916.210521:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767916.210524:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767916.210531:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055f5621a] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8380 x1862022294251392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767916.210549:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767916.210554:0:13471:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5178) 00000800:00000200:0.1:1775767916.210687:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767916.210689:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775767916.210893:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767916.210895:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775767916.211099:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767916.211101:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775767916.211338:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767916.211341:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000100:00000001:1.0:1775767917.233302:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767917.233302:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767917.233306:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767917.233310:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767917.233310:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767917.233311:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767917.233314:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767917.233315:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767917.233315:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767917.233323:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:1.0:1775767917.233324:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775767917.233324:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767917.233330:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767917.233330:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767917.233331:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767917.233336:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767917.233336:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767917.233337:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767917.233341:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767917.233344:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767917.233346:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767917.233349:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767917.233351:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767917.233354:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767917.233354:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767917.233357:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767917.233360:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767917.553252:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299842366, at 4299842368 00000100:00000010:1.0:1775767917.553270:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3660. 00000100:00000001:1.0:1775767918.257264:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767918.257270:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767918.257271:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767918.257272:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767918.257278:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767918.257279:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767918.257279:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767918.257288:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767918.257290:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767918.257290:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767918.257303:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767918.257305:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1775767918.257305:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775767918.257307:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767918.257313:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767918.257315:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767918.257316:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767918.257318:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767918.257324:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767918.257324:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767918.257339:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767918.257339:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767918.257343:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767918.257346:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767918.257347:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767918.257350:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767918.257353:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767919.281317:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767919.281317:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767919.281331:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767919.281332:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767919.281340:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767919.281340:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767919.281354:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775767919.281354:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767919.281365:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767919.281366:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767919.281374:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767919.281375:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767919.281385:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767919.281388:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767919.281391:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767919.281393:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767919.281397:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767919.281403:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767920.305231:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767920.305239:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767920.305241:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767920.305248:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767920.305253:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767920.305254:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767920.305257:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767920.305261:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767920.305266:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767920.305266:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767920.305270:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767920.305272:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767920.305275:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767920.305280:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767920.305285:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767920.305291:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767920.305294:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767920.305298:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767921.329290:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:3.0:1775767921.329291:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775767921.329299:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:3.0:1775767921.329301:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775767921.329302:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767921.329305:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:1.0:1775767921.329308:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00080000:3.0:1775767921.329309:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775767921.329316:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767921.329317:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775767921.329320:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767921.329326:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:3.0:1775767921.329327:0:13471:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5173 00000100:00000001:1.0:1775767921.329330:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767921.329337:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767921.329337:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000400:00000200:3.0:1775767921.329347:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767921.329348:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767921.329355:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767921.329360:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00100000:2.0:1775767921.329363:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00080000:3.0:1775767921.329364:0:13471:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1775767921.329372:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1775767921.329374:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767921.329378:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767921.329378:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775767921.329386:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000040:3.0:1775767921.329386:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767921.329391:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767921.329392:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:2.0:1775767921.329401:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767921.329402:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767921.329408:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:3.0:1775767921.329413:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767921.329416:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:3.0:1775767921.329420:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:3.0:1775767921.329422:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:3.0:1775767921.329426:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767921.329428:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767921.329431:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767921.329433:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767921.329435:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:3.0:1775767921.329438:0:13471:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767921.329439:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767921.329442:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767921.329444:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767921.329447:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767921.329449:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775767921.329452:0:13471:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767921.329462:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8380. 00000020:00000040:3.0:1775767921.329466:0:13471:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:3.0:1775767921.329469:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767921.329470:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767921.329473:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767921.329476:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767921.329482:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767921.329491:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0000. 00000100:00000001:3.0:1775767921.329498:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767921.329507:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8380 x1862022294251520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767921.329517:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767921.329522:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767921.329528:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055f5621a] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8380 x1862022294251520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767921.329551:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767921.329555:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767921.329559:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775767921.329560:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767921.329562:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767921.329565:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767921.329569:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5173 00000400:00000200:3.0:1775767921.329574:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767921.329577:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767921.329579:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767921.329583:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767921.329587:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767921.329590:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767921.329592:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767921.329595:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767921.329598:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:3.0:1775767921.329601:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767921.329605:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:3.0:1775767921.329610:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:3.0:1775767921.329613:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:3.0:1775767921.329617:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767921.329620:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767921.329623:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767921.329625:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767921.329628:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00080000:3.0:1775767921.329633:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767921.329637:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767921.329638:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767921.329644:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767921.329646:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767921.329651:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767921.329670:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775767921.329676:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767921.329679:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767921.329689:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8700. 00000020:00000040:3.0:1775767921.329695:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:3.0:1775767921.329699:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767921.329702:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767921.329705:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767921.329709:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767921.329712:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767921.329718:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2800. 00000100:00000001:3.0:1775767921.329724:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767921.329734:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8700 x1862022294251648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767921.329751:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767921.329755:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767921.329761:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000faa25e3b] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8700 x1862022294251648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767921.329780:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767921.329783:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767921.329788:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775767921.329791:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767921.329793:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767921.329795:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767921.329801:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5173 00000400:00000200:3.0:1775767921.329807:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767921.329811:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767921.329812:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767921.329815:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767921.329816:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767921.329819:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767921.329823:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767921.329824:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767921.329826:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:3.0:1775767921.329828:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767921.329832:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:3.0:1775767921.329834:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:3.0:1775767921.329837:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:3.0:1775767921.329844:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767921.329847:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767921.329851:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767921.329854:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767921.329857:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:3.0:1775767921.329863:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767921.329866:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767921.329870:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767921.329875:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767921.329877:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767921.329879:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767921.329881:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775767921.329886:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767921.329890:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767921.329894:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a9180. 00000020:00000040:3.0:1775767921.329897:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:3.0:1775767921.329900:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767921.329902:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767921.329904:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767921.329907:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767921.329909:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767921.329913:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0c00. 00000100:00000001:3.0:1775767921.329919:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767921.329923:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a9180 x1862022294251776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767921.329936:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767921.329938:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767921.329944:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bdc2586] to pc [ptlrpcd_rcv+-1] req@ffff9942099a9180 x1862022294251776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.1:1775767921.329951:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:3.0:1775767921.329955:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775767921.329957:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000100:00080000:3.0:1775767921.329958:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767921.329965:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775767921.329966:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767921.329968:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767921.329972:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767921.329976:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5173 00000400:00000200:3.0:1775767921.329981:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767921.329983:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767921.329984:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767921.329987:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767921.329989:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767921.329992:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767921.329994:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767921.329998:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767921.330000:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:3.0:1775767921.330002:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767921.330006:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:3.0:1775767921.330007:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:3.0:1775767921.330009:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:3.0:1775767921.330013:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767921.330016:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767921.330020:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767921.330022:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767921.330025:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:3.0:1775767921.330030:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767921.330036:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767921.330040:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767921.330043:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767921.330046:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767921.330049:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767921.330052:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:3.0:1775767921.330057:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767921.330061:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767921.330066:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ab480. 00000020:00000040:3.0:1775767921.330069:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:3.0:1775767921.330074:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767921.330075:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767921.330076:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767921.330078:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767921.330080:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767921.330085:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2000. 00000100:00000001:3.0:1775767921.330090:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767921.330105:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ab480 x1862022294251904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767921.330114:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767921.330117:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767921.330121:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000067a0ba74] to pc [ptlrpcd_rcv+-1] req@ffff9942099ab480 x1862022294251904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767921.330133:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767921.330138:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767921.330142:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775767921.330145:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767921.330148:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767921.330153:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767921.330176:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5173 00000400:00000200:3.0:1775767921.330180:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767921.330185:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767921.330187:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767921.330193:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767921.330196:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767921.330200:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767921.330203:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767921.330206:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767921.330208:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:3.0:1775767921.330210:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767921.330212:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:3.0:1775767921.330217:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:3.0:1775767921.330223:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:3.0:1775767921.330230:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767921.330233:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767921.330238:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767921.330243:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767921.330248:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:3.0:1775767921.330252:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767921.330258:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767921.330262:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767921.330265:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767921.330268:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767921.330274:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767921.330277:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:3.0:1775767921.330282:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767921.330286:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767921.330291:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8e00. 00000020:00000040:3.0:1775767921.330295:0:13471:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:3.0:1775767921.330301:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767921.330303:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767921.330306:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767921.330308:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767921.330310:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767921.330315:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0400. 00000100:00000001:3.0:1775767921.330319:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767921.330328:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8e00 x1862022294252032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767921.330344:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767921.330348:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767921.330356:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000864a6866] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8e00 x1862022294252032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767921.330372:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767921.330377:0:13471:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5183) 00000800:00000200:0.1:1775767921.330512:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767921.330515:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775767921.330756:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767921.330758:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775767921.330974:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767921.330976:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775767921.331170:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767921.331173:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000100:00000001:1.0:1775767922.353246:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767922.353255:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767922.353257:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767922.353265:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767922.353273:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767922.353278:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767922.353283:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767922.353284:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767922.353286:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767922.353292:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767922.353293:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767922.353302:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767922.353308:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775767922.353315:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767922.353318:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767922.353320:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767922.353323:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767922.353326:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767922.673225:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299847485, at 4299847488 00000100:00000010:1.0:1775767922.673243:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3f60. 00000100:00000001:2.0:1775767923.377237:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767923.377239:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767923.377246:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767923.377248:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767923.377254:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775767923.377255:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767923.377259:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767923.377261:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:2.0:1775767923.377266:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767923.377267:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767923.377273:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767923.377274:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767923.377279:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767923.377279:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767923.377287:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767923.377287:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767923.377291:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767923.377295:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767924.401257:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767924.401258:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767924.401263:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767924.401264:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767924.401265:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767924.401268:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767924.401269:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:2.0:1775767924.401275:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767924.401275:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767924.401278:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767924.401280:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767924.401282:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767924.401284:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767924.401285:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767924.401286:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767924.401290:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767924.401294:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767924.401297:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767925.425246:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767925.425247:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767925.425254:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767925.425254:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767925.425256:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767925.425258:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767925.425263:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775767925.425264:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775767925.425268:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767925.425270:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767925.425272:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767925.425274:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767925.425277:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767925.425280:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767925.425284:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767925.425287:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767925.425297:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767925.425301:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767926.449265:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767926.449266:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767926.449272:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767926.449273:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767926.449276:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767926.449276:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767926.449283:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775767926.449283:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775767926.449289:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767926.449289:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767926.449295:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767926.449295:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767926.449301:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767926.449304:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767926.449308:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767926.449308:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767926.449313:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767926.449316:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775767926.449349:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767926.449354:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775767926.449357:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775767926.449359:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775767926.449363:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775767926.449369:0:13471:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5178 00000400:00000200:2.0:1775767926.449379:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767926.449385:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775767926.449387:0:13471:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775767926.449391:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767926.449394:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767926.449396:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767926.449399:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775767926.449402:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775767926.449412:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:2.0:1775767926.449414:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775767926.449419:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:2.0:1775767926.449420:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:2.0:1775767926.449423:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:2.0:1775767926.449426:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767926.449427:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767926.449429:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767926.449431:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775767926.449433:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:2.0:1775767926.449435:0:13471:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775767926.449436:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767926.449439:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775767926.449441:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767926.449443:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767926.449445:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775767926.449447:0:13471:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775767926.449455:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfdc00. 00000020:00000040:2.0:1775767926.449457:0:13471:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:2.0:1775767926.449461:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775767926.449462:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767926.449464:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775767926.449466:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775767926.449470:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775767926.449477:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1ec00. 00000100:00000001:2.0:1775767926.449482:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767926.449489:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfdc00 x1862022294252160/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775767926.449497:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775767926.449500:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767926.449504:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003e25603c] to pc [ptlrpcd_rcv+-1] req@ffff994207dfdc00 x1862022294252160/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767926.449519:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775767926.449521:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767926.449524:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775767926.449525:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775767926.449527:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775767926.449529:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775767926.449530:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5178 00000400:00000200:2.0:1775767926.449533:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767926.449536:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775767926.449538:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775767926.449540:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767926.449543:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767926.449544:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767926.449546:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775767926.449548:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775767926.449550:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:2.0:1775767926.449551:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775767926.449553:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:2.0:1775767926.449554:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:2.0:1775767926.449557:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:2.0:1775767926.449561:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767926.449563:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767926.449566:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767926.449568:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775767926.449569:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00080000:2.0:1775767926.449573:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775767926.449575:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767926.449577:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775767926.449582:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767926.449586:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767926.449589:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775767926.449594:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775767926.449597:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767926.449599:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775767926.449603:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dff100. 00000020:00000040:2.0:1775767926.449629:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:2.0:1775767926.449633:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775767926.449634:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767926.449638:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775767926.449640:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775767926.449643:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775767926.449647:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1c800. 00000100:00000001:2.0:1775767926.449651:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767926.449658:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dff100 x1862022294252288/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775767926.449665:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775767926.449669:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767926.449673:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3e64ae1] to pc [ptlrpcd_rcv+-1] req@ffff994207dff100 x1862022294252288/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767926.449683:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775767926.449686:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767926.449688:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775767926.449691:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775767926.449692:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775767926.449695:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775767926.449697:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5178 00000400:00000200:2.0:1775767926.449702:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767926.449704:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775767926.449706:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775767926.449711:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767926.449714:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767926.449717:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767926.449719:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775767926.449721:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775767926.449723:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:2.0:1775767926.449725:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775767926.449726:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:2.0:1775767926.449729:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:2.0:1775767926.449733:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:2.0:1775767926.449738:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767926.449741:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767926.449744:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767926.449748:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775767926.449751:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:2.0:1775767926.449754:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775767926.449758:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767926.449759:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775767926.449762:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767926.449765:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767926.449767:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775767926.449771:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775767926.449774:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767926.449777:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775767926.449781:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfc700. 00000020:00000040:2.0:1775767926.449784:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:2.0:1775767926.449786:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775767926.449789:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767926.449794:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775767926.449797:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775767926.449799:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775767926.449806:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1d400. 00000100:00000001:2.0:1775767926.449812:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767926.449822:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfc700 x1862022294252416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775767926.449836:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775767926.449841:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767926.449850:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a176dd1] to pc [ptlrpcd_rcv+-1] req@ffff994207dfc700 x1862022294252416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767926.449863:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775767926.449865:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767926.449870:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1775767926.449870:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775767926.449872:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775767926.449873:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775767926.449876:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5178 00000400:00000200:2.0:1775767926.449881:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767926.449887:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775767926.449890:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775767926.449896:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767926.449898:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767926.449904:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767926.449909:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775767926.449911:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775767926.449912:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:2.0:1775767926.449915:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775767926.449917:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:2.0:1775767926.449919:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:2.0:1775767926.449920:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:2.0:1775767926.449924:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767926.449926:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767926.449929:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767926.449931:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775767926.449933:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:2.0:1775767926.449937:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775767926.449941:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767926.449943:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:0.1:1775767926.449945:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 02000000:00000001:2.0:1775767926.449947:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767926.449951:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775767926.449951:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000100:00000001:2.0:1775767926.449953:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775767926.449956:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:2.0:1775767926.449960:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767926.449962:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775767926.449967:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfe300. 00000020:00000040:2.0:1775767926.449971:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:2.0:1775767926.449973:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775767926.449973:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767926.449975:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775767926.449977:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775767926.449980:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775767926.449985:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1fc00. 00000100:00000001:2.0:1775767926.449990:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767926.449995:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfe300 x1862022294252544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775767926.450004:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775767926.450006:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767926.450011:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c661fef] to pc [ptlrpcd_rcv+-1] req@ffff994207dfe300 x1862022294252544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767926.450018:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775767926.450021:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767926.450026:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1775767926.450028:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775767926.450029:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775767926.450035:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775767926.450039:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5178 00000400:00000200:2.0:1775767926.450043:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767926.450046:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775767926.450048:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775767926.450053:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767926.450058:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767926.450062:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767926.450066:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775767926.450070:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775767926.450072:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:2.0:1775767926.450075:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775767926.450077:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:2.0:1775767926.450080:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:2.0:1775767926.450081:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:2.0:1775767926.450084:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767926.450087:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767926.450090:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767926.450092:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775767926.450094:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:2.0:1775767926.450099:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775767926.450103:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767926.450105:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775767926.450108:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767926.450111:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767926.450112:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775767926.450113:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:2.0:1775767926.450115:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767926.450116:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775767926.450122:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfed80. 00000020:00000040:2.0:1775767926.450123:0:13471:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:2.0:1775767926.450125:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775767926.450127:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767926.450130:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775767926.450131:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775767926.450133:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775767926.450136:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1f400. 00000100:00000001:2.0:1775767926.450140:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767926.450145:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfed80 x1862022294252672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775767926.450174:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775767926.450176:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767926.450179:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb9b9eae] to pc [ptlrpcd_rcv+-1] req@ffff994207dfed80 x1862022294252672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767926.450186:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767926.450191:0:13471:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5188) 00000800:00000200:0.1:1775767926.450499:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767926.450501:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 02000000:00000001:0.0:1775767926.450702:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767926.450705:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294252416:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775767926.450708:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775767926.450711:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775767926.450712:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767926.450714:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775767926.450719:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220029400. 02000000:00000001:0.0:1775767926.450721:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767926.450725:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775767926.450729:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775767926.450733:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775767926.450736:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294252416, portal 10 00000100:00000001:0.0:1775767926.450738:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775767926.450739:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611838208 : -112966097713408 : ffff994207dfc700) 00000100:00000040:0.0:1775767926.450744:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfc700 x1862022294252416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767951 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767926.450750:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775767926.450752:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775767926.450755:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775767926.450757:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294252416, offset 0 00000400:00000200:0.0:1775767926.450761:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775767926.450765:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767926.450768:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775767926.450771:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775767926.450774:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775767926.450776:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775767926.450778:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775767926.450782:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225145:225145:256:4294967295] 192.168.202.156@tcp LPNI seq info [225141:225141:8:4294967295] 00000400:00000200:0.0:1775767926.450786:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767926.450792:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775767926.450797:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767926.450800:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775767926.450804:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775767926.450807:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775767926.450811:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767926.450813:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1775767926.450817:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767926.450819:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775767926.450822:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775767926.450825:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775767926.450828:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775767926.450829:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775767926.450832:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775767926.450835:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfc700 x1862022294252416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767951 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767926.450842:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767926.450846:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfc700 x1862022294252416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767951 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767926.450853:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767926.450855:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775767926.450859:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767926.450863:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:0.0:1775767926.450866:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775767926.450868:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299851275 00000100:00000001:0.0:1775767926.450870:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299852605 00000100:00000001:0.0:1775767926.450872:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299851275 00000100:00000001:0.0:1775767926.450874:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299852605 00000100:00000001:0.0:1775767926.450876:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775767926.450878:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767926.450880:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775767926.450884:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767926/real 1775767926] req@ffff994207dfc700 x1862022294252416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775767926.450891:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775767926.450893:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775767926.450894:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775767926.450896:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775767926.450898:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775767926.450900:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775767926.450903:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfc700 x1862022294252416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775767926.450911:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfc700 x1862022294252416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767926.450917:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775767926.450919:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767926.450920:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767926.450922:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775767926.450925:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfc700 x1862022294252416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767926.450932:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775767926.450936:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfc700 x1862022294252416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767951 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767926.450942:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775767926.450944:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767926.450946:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767926.450947:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767926.450949:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775767926.450952:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775767926.450954:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767926.450958:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775767926.450960:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775767926.450964:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfc700 x1862022294252416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767951 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767926.450979:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294252416:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775767926.450984:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775767926.450988:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfe300 x1862022294252544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767926.450994:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767926.450995:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767926.450997:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775767926.450999:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775767926.451001:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767926.451004:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294252544:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775767926.451007:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775767926.451008:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775767926.451010:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767926.451012:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775767926.451014:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002b800. 02000000:00000001:0.0:1775767926.451016:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767926.451020:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775767926.451023:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775767926.451026:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775767926.451028:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294252544, portal 4 00000100:00000001:0.0:1775767926.451031:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775767926.451032:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611845376 : -112966097706240 : ffff994207dfe300) 00000100:00000040:0.0:1775767926.451037:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfe300 x1862022294252544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767951 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767926.451043:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775767926.451046:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775767926.451048:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775767926.451050:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294252544, offset 0 00000400:00000200:0.0:1775767926.451054:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775767926.451058:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767926.451061:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775767926.451064:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775767926.451066:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775767926.451069:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775767926.451071:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775767926.451075:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225146:225146:256:4294967295] 192.168.202.156@tcp LPNI seq info [225142:225142:8:4294967295] 00000400:00000200:0.0:1775767926.451079:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767926.451084:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775767926.451088:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767926.451091:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775767926.451095:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775767926.451098:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775767926.451102:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767926.451105:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1775767926.451108:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767926.451111:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775767926.451113:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775767926.451117:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775767926.451120:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775767926.451122:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775767926.451124:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775767926.451128:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfe300 x1862022294252544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767951 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767926.451136:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767926.451140:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfe300 x1862022294252544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767951 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767926.451147:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767926.451149:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775767926.451171:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767926.451174:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:0.0:1775767926.451178:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775767926.451181:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299851299 00000100:00000001:0.0:1775767926.451183:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299852605 00000100:00000001:0.0:1775767926.451185:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299851299 00000100:00000001:0.0:1775767926.451187:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299852605 00000100:00000001:0.0:1775767926.451189:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775767926.451191:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767926.451193:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775767926.451197:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767926/real 1775767926] req@ffff994207dfe300 x1862022294252544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775767926.451204:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775767926.451206:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775767926.451207:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775767926.451209:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775767926.451211:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775767926.451213:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775767926.451216:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfe300 x1862022294252544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775767926.451224:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfe300 x1862022294252544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767926.451230:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775767926.451232:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767926.451233:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767926.451235:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775767926.451238:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfe300 x1862022294252544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767926.451244:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775767926.451248:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfe300 x1862022294252544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767951 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767926.451254:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775767926.451257:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767926.451258:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767926.451259:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767926.451261:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775767926.451265:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775767926.451267:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767926.451270:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775767926.451273:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775767926.451276:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfe300 x1862022294252544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767951 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767926.451284:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294252544:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775767926.451287:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775767926.451290:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfed80 x1862022294252672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767926.451296:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767926.451298:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767926.451300:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775767926.451302:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775767926.451304:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767926.451306:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294252672:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775767926.451309:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775767926.451311:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775767926.451313:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767926.451315:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775767926.451318:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220028000. 02000000:00000001:0.0:1775767926.451319:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767926.451322:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775767926.451324:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775767926.451327:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775767926.451330:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294252672, portal 4 00000100:00000001:0.0:1775767926.451332:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775767926.451333:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611848064 : -112966097703552 : ffff994207dfed80) 00000100:00000040:0.0:1775767926.451338:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfed80 x1862022294252672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767951 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767926.451344:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775767926.451346:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775767926.451349:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775767926.451351:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294252672, offset 0 00000400:00000200:0.0:1775767926.451355:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775767926.451358:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767926.451361:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775767926.451364:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775767926.451366:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775767926.451369:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775767926.451371:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775767926.451374:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225147:225147:256:4294967295] 192.168.202.156@tcp LPNI seq info [225143:225143:8:4294967295] 00000400:00000200:0.0:1775767926.451378:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767926.451384:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775767926.451388:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767926.451391:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775767926.451394:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775767926.451397:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775767926.451401:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767926.451403:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1775767926.451407:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767926.451409:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775767926.451412:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775767926.451415:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775767926.451418:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775767926.451419:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775767926.451422:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775767926.451425:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfed80 x1862022294252672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767951 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767926.451437:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767926.451440:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfed80 x1862022294252672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767951 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767926.451446:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767926.451448:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775767926.451450:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767926.451453:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:0.0:1775767926.451456:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775767926.451458:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299851302 00000100:00000001:0.0:1775767926.451460:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299852605 00000100:00000001:0.0:1775767926.451462:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299851302 00000100:00000001:0.0:1775767926.451464:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299852605 00000100:00000001:0.0:1775767926.451466:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775767926.451468:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767926.451470:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775767926.451473:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767926/real 1775767926] req@ffff994207dfed80 x1862022294252672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775767926.451480:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775767926.451481:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775767926.451483:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775767926.451485:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775767926.451486:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775767926.451489:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775767926.451492:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfed80 x1862022294252672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775767926.451499:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfed80 x1862022294252672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767926.451505:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775767926.451507:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767926.451509:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767926.451510:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775767926.451513:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfed80 x1862022294252672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767926.451520:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775767926.451523:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfed80 x1862022294252672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767951 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767926.451530:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775767926.451532:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767926.451534:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767926.451535:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767926.451537:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775767926.451540:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775767926.451542:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767926.451545:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775767926.451547:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775767926.451551:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfed80 x1862022294252672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767951 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767926.451558:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294252672:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775767926.451562:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767926.451563:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767926.451566:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dff100 x1862022294252288/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767951 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767926.451572:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775767926.451574:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775767926.451576:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220029800. 02000000:00000001:0.0:1775767926.451578:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775767926.451580:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775767926.451581:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:0.0:1775767926.451583:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775767926.451585:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1c800. 02000000:00000001:0.0:1775767926.451587:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775767926.451589:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775767926.451590:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dff100. 00000100:00000001:0.0:1775767926.451593:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775767926.451594:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767926.451596:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767926.451599:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfc700 x1862022294252416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767951 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767926.451623:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775767926.451624:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775767926.451626:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220029400. 02000000:00000001:0.0:1775767926.451629:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775767926.451630:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775767926.451631:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:0.0:1775767926.451633:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775767926.451635:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1d400. 02000000:00000001:0.0:1775767926.451638:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775767926.451639:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775767926.451641:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfc700. 00000100:00000001:0.0:1775767926.451643:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775767926.451644:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767926.451646:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767926.451649:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfe300 x1862022294252544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767951 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767926.451656:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775767926.451657:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775767926.451658:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002b800. 02000000:00000001:0.0:1775767926.451661:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775767926.451662:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775767926.451663:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:0.0:1775767926.451665:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775767926.451666:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1fc00. 02000000:00000001:0.0:1775767926.451669:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775767926.451670:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775767926.451672:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfe300. 00000100:00000001:0.0:1775767926.451674:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775767926.451675:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767926.451677:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767926.451680:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfed80 x1862022294252672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767951 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767926.451686:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775767926.451687:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775767926.451689:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220028000. 02000000:00000001:0.0:1775767926.451691:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775767926.451692:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775767926.451693:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:0.0:1775767926.451695:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775767926.451696:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1f400. 02000000:00000001:0.0:1775767926.451698:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775767926.451700:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775767926.451701:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfed80. 00000100:00000001:0.0:1775767926.451703:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775767926.451704:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767926.451706:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767926.451710:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767926.451711:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767926.451713:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775767926.451715:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767926.513268:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767926.513275:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767926.513281:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767926.513283:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767926.513287:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767926.513289:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767926.513292:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767926.513294:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767926.513297:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767926.513299:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767926.513319:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767926.513322:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767926.513325:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767926.577271:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767926.577277:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767926.577278:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767926.577283:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767926.577286:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767926.577289:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767926.577291:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767926.577293:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767926.577295:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767927.473239:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767927.473239:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767927.473247:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767927.473247:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767927.473251:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767927.473251:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767927.473258:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:2.0:1775767927.473258:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767927.473262:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767927.473263:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767927.473264:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767927.473269:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767927.473270:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767927.473270:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767927.473273:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767927.473277:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00100000:1.0:1775767927.473278:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:0.0:1775767927.473281:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767927.473282:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767927.473287:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767927.473288:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767927.473288:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767927.473293:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767927.473297:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767927.473301:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767927.473305:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767927.473308:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767927.537264:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767927.537269:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767927.537274:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767927.537275:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767927.537279:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767927.537279:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767927.537282:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767927.537283:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767927.537285:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767927.537286:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767927.537302:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767927.537304:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767927.537306:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767927.601206:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767927.601212:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767927.601214:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767927.601218:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767927.601221:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767927.601224:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767927.601227:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767927.601229:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767927.601231:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767927.793257:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299852605, at 4299852608 00000100:00000040:0.0:1775767927.793259:0:14953:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4299852605, at 4299852608 00000100:00000010:1.0:1775767927.793273:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3cc0. 00000100:00000010:0.0:1775767927.793276:0:14953:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3d80. 00000100:00000001:1.0:1775767928.497236:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767928.497238:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767928.497243:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767928.497244:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767928.497245:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767928.497247:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767928.497252:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:0.0:1775767928.497253:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775767928.497260:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767928.497260:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767928.497265:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767928.497266:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767928.497272:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767928.497275:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767928.497278:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767928.497279:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767928.497281:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767928.497284:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767928.497316:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767928.497324:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767928.497326:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767928.497339:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775767928.497342:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767928.497346:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767928.497348:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767928.497350:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767928.497352:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767928.561304:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767928.561311:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767928.561316:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767928.561318:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767928.561321:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767928.561323:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767928.561326:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767928.561327:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767928.561330:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767928.561331:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767928.561352:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767928.561354:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767928.561357:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767928.625257:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767928.625264:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767928.625266:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767928.625273:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767928.625277:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767928.625281:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767928.625283:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767928.625286:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767928.625288:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767929.521232:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767929.521238:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767929.521241:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767929.521244:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767929.521245:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767929.521248:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767929.521251:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:0.0:1775767929.521253:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775767929.521257:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767929.521257:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767929.521264:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767929.521264:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767929.521270:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767929.521272:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767929.521276:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767929.521278:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767929.521280:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767929.521284:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767929.521308:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767929.521313:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767929.521315:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767929.521319:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767929.521323:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767929.521326:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767929.521329:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767929.521331:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767929.521334:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767929.585266:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767929.585272:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767929.585278:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767929.585280:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767929.585283:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767929.585284:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767929.585287:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767929.585288:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767929.585290:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767929.585291:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767929.585310:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767929.585312:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767929.585315:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767929.649286:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767929.649293:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767929.649295:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767929.649301:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767929.649305:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767929.649309:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767929.649312:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767929.649314:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767929.649317:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767930.545254:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767930.545258:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767930.545262:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767930.545264:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767930.545266:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767930.545267:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767930.545267:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767930.545273:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:1.0:1775767930.545275:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:0.0:1775767930.545278:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767930.545278:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767930.545281:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:0.0:1775767930.545288:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767930.545288:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767930.545291:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767930.545294:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767930.545298:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767930.545298:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767930.545298:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767930.545305:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767930.545306:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767930.545306:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767930.545314:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767930.545319:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767930.545323:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767930.545326:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767930.545330:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767930.609261:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767930.609268:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775767930.609270:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775767930.609273:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767930.609277:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775767930.609279:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767930.609280:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767930.609282:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767930.609284:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767930.609285:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767930.609287:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767930.609288:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767930.609290:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767930.609291:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767930.609293:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767930.609294:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767930.609309:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767930.609311:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767930.609314:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767930.673226:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767930.673231:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767930.673233:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767930.673240:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767930.673243:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767930.673247:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767930.673250:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767930.673253:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767930.673256:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767931.569227:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767931.569236:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767931.569239:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767931.569247:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767931.569253:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767931.569258:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767931.569262:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767931.569266:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767931.569267:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767931.569267:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767931.569273:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767931.569273:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767931.569274:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767931.569276:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767931.569278:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767931.569280:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:1.0:1775767931.569283:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:0.0:1775767931.569285:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767931.569290:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767931.569290:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767931.569292:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767931.569294:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767931.569295:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767931.569296:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767931.569300:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767931.569304:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767931.569310:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775767931.569325:0:13471:0:(rw.c:1481:ll_writepages()) Process entered 00000080:00000001:2.0:1775767931.569331:0:13471:0:(file.c:5250:cl_sync_file_range()) Process entered 00000020:00000001:2.0:1775767931.569334:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775767931.569346:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994210b32630. 00000100:00080000:1.0:1775767931.569354:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000010:2.0:1775767931.569356:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9942060d5a00. 00000100:00080000:1.0:1775767931.569365:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000020:00000010:2.0:1775767931.569366:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff9942060d4c00. 00000100:00000001:1.0:1775767931.569370:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000020:00000010:2.0:1775767931.569373:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942182d6c00. 00000100:00080000:1.0:1775767931.569375:0:8:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775767931.569381:0:8:0:(import.c:484:import_select_connection()) Process entered 00000008:00000010:2.0:1775767931.569382:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207ccc2f0. 00020000:00000010:2.0:1775767931.569389:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994220156e40. 00000100:00080000:1.0:1775767931.569390:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5183 00008000:00000010:2.0:1775767931.569399:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942077a8e20. 00000400:00000200:1.0:1775767931.569404:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000080:00000010:2.0:1775767931.569405:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff9942087472a0. 00000080:00000010:2.0:1775767931.569413:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b8000. 00000020:00000010:2.0:1775767931.569416:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9942060d5800. 00000400:00000200:1.0:1775767931.569421:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775767931.569428:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000008:00000010:2.0:1775767931.569431:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942077cd600. 00000100:00000040:1.0:1775767931.569435:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00020000:00000010:2.0:1775767931.569437:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff99420876d680. 00000100:00000001:1.0:1775767931.569445:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00008000:00000010:2.0:1775767931.569446:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076cf60. 00000080:00000010:2.0:1775767931.569452:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff994205066870. 00000100:00000040:1.0:1775767931.569453:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000001:2.0:1775767931.569459:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107759908408 : -112965949643208 : ffff994210b32638) 00000100:00000001:1.0:1775767931.569460:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00001000:2.0:1775767931.569467:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff994210b32630 00000020:00000001:1.0:1775767931.569469:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000001:2.0:1775767931.569472:0:13471:0:(cl_object.c:151:cl_object_top()) ffff99420791a5f8 -> ffff99420791a5f8 00000020:00000040:1.0:1775767931.569473:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:2.0:1775767931.569476:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:2.0:1775767931.569482:0:13471:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000020:00000001:1.0:1775767931.569483:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000080:00200000:2.0:1775767931.569484:0:13471:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x10176:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000040:1.0:1775767931.569488:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:2.0:1775767931.569489:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:1.0:1775767931.569496:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:2.0:1775767931.569496:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:2.0:1775767931.569499:0:13471:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775767931.569501:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00020000:00000002:2.0:1775767931.569504:0:13471:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x10176:0x0]io ffff9942077b8060 type 6 ignore/verify layout 1/0 00000100:00000040:1.0:1775767931.569508:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00020000:00000002:2.0:1775767931.569511:0:13471:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207dfb6c8) owner 0000000000000000/ffff994220080000 00000100:00000001:1.0:1775767931.569513:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00020000:00000001:2.0:1775767931.569516:0:13471:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00000100:00000040:1.0:1775767931.569522:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00020000:00000001:2.0:1775767931.569522:0:13471:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:2.0:1775767931.569525:0:13471:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00020000:00000001:2.0:1775767931.569530:0:13471:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767931.569531:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00020000:00000001:2.0:1775767931.569533:0:13471:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:2.0:1775767931.569541:0:13471:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00000020:00000040:1.0:1775767931.569543:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00020000:00000001:2.0:1775767931.569543:0:13471:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767931.569552:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775767931.569554:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00080000:1.0:1775767931.569567:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00020000:00000001:2.0:1775767931.569572:0:13471:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00000100:00000001:1.0:1775767931.569576:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775767931.569577:0:13471:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207dfb6c8) owner 0000000000000000/ffff994220080000 00000020:00000001:2.0:1775767931.569582:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767931.569583:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000020:00000001:2.0:1775767931.569586:0:13471:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:2.0:1775767931.569589:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 02000000:00000001:1.0:1775767931.569590:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:2.0:1775767931.569592:0:13471:0:(lov_io.c:836:lov_io_iter_init()) Process entered 02000000:00000001:1.0:1775767931.569597:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00020000:00200000:2.0:1775767931.569598:0:13471:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00000001:1.0:1775767931.569601:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775767931.569605:0:8:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775767931.569605:0:13471:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:2.0:1775767931.569615:0:13471:0:(lov_io.c:155:lov_sub_get()) Process entered 00020000:00000001:2.0:1775767931.569617:0:13471:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000020:00000001:2.0:1775767931.569619:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00000010:1.0:1775767931.569621:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207888700. 00000020:00000010:2.0:1775767931.569624:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994210b32090. 00000020:00000040:1.0:1775767931.569629:0:8:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 00000020:00000010:2.0:1775767931.569631:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9942060d5200. 02000000:00000001:1.0:1775767931.569635:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000020:00000010:2.0:1775767931.569637:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff9942060d4800. 02000000:00000001:1.0:1775767931.569638:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767931.569642:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000020:00000010:2.0:1775767931.569644:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942182d6cc0. 02000000:00000001:1.0:1775767931.569646:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000008:00000010:2.0:1775767931.569648:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207cc8b28. 00020000:00000010:2.0:1775767931.569651:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994220156840. 02000000:00000001:1.0:1775767931.569652:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:2.0:1775767931.569655:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942077a9530. 00000080:00000010:2.0:1775767931.569658:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994208747738. 02000000:00000010:1.0:1775767931.569664:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543c800. 00000080:00000010:2.0:1775767931.569664:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077ba910. 00000020:00000010:2.0:1775767931.569675:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994209d2cc00. 00000100:00000001:1.0:1775767931.569676:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:2.0:1775767931.569681:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942077cd8c0. 00020000:00000010:2.0:1775767931.569684:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff99420876e400. 00008000:00000010:2.0:1775767931.569686:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076c3f0. 00000080:00000010:2.0:1775767931.569689:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942050663c0. 00000020:00000001:2.0:1775767931.569695:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107759906968 : -112965949644648 : ffff994210b32098) 00000100:00100000:1.0:1775767931.569696:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207888700 x1862022294252800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00001000:2.0:1775767931.569698:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff994210b32090 00000020:00000001:2.0:1775767931.569705:0:13471:0:(cl_object.c:151:cl_object_top()) ffff99420791ae90 -> ffff99420791a5f8 00000020:00000001:2.0:1775767931.569709:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:2.0:1775767931.569712:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775767931.569716:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:2.0:1775767931.569720:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767931.569728:0:13471:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767931.569735:0:13471:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107621738216 : -112966087813400 : ffff99420876d6e8) 00000100:00000001:1.0:1775767931.569738:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000020:00000001:2.0:1775767931.569738:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000020:00000001:2.0:1775767931.569739:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767931.569745:0:13471:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767931.569748:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767931.569749:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767931.569751:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00020000:00000001:2.0:1775767931.569756:0:13471:0:(lov_io.c:1108:lov_io_lock()) Process entered 00020000:00000001:2.0:1775767931.569758:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775767931.569760:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:2.0:1775767931.569764:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000100:00000040:1.0:1775767931.569765:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009f2ba301] to pc [ptlrpcd_rcv+-1] req@ffff994207888700 x1862022294252800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1775767931.569767:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767931.569769:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767931.569775:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767931.569782:0:13471:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767931.569790:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:2.0:1775767931.569796:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767931.569805:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767931.569810:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00020000:00000001:2.0:1775767931.569811:0:13471:0:(lov_io.c:1114:lov_io_start()) Process entered 00020000:00000001:2.0:1775767931.569814:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775767931.569815:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00000008:00000001:2.0:1775767931.569817:0:13471:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000008:00000001:2.0:1775767931.569820:0:13471:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000100:00000001:1.0:1775767931.569826:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1775767931.569826:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:2.0:1775767931.569832:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000100:00080000:1.0:1775767931.569834:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000008:00000001:2.0:1775767931.569835:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1775767931.569846:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000008:00000001:2.0:1775767931.569847:0:13471:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000100:00000001:1.0:1775767931.569848:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775767931.569853:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000008:00000001:2.0:1775767931.569854:0:13471:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000100:00000001:0.0:1775767931.569859:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00000001:2.0:1775767931.569859:0:13471:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107754940544 : -112965954611072 : ffff994210675880) 00000100:00000001:1.0:1775767931.569860:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:0.0:1775767931.569863:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:1.0:1775767931.569866:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5183 00000100:00000001:0.0:1775767931.569867:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000008:00000020:2.0:1775767931.569873:0:13471:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994210675880 ready 0|-|- wr 1|-|+ rd 0|- 10 in flight 00000400:00000200:1.0:1775767931.569880:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000040:0.0:1775767931.569882:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207888700 x1862022294252800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775767931.569892:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000008:00000001:2.0:1775767931.569893:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:2.0:1775767931.569899:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000100:00080000:1.0:1775767931.569904:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000008:00000001:2.0:1775767931.569906:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1775767931.569917:0:13471:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000100:00000001:0.0:1775767931.569918:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000040:1.0:1775767931.569922:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775767931.569923:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000020:2.0:1775767931.569925:0:13471:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994218232fc0 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 02000000:00000001:0.0:1775767931.569929:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1775767931.569932:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1775767931.569937:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:2.0:1775767931.569940:0:13471:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775767931.569941:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000001:0.0:1775767931.569943:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1775767931.569950:0:13471:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767931.569951:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00100000:0.0:1775767931.569956:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207888700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294252800:192.168.202.156@tcp:250:kworker.0 00000020:00000001:2.0:1775767931.569957:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767931.569965:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775767931.569966:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:0.0:1775767931.569973:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000040:1.0:1775767931.569974:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00020000:00000001:2.0:1775767931.569975:0:13471:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767931.569977:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:1.0:1775767931.569979:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000001:2.0:1775767931.569984:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767931.569985:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1775767931.569985:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:2.0:1775767931.569994:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 02000000:00000001:0.0:1775767931.569995:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:1.0:1775767931.569995:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00020000:00000001:2.0:1775767931.570000:0:13471:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00000020:00000001:1.0:1775767931.570002:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 02000000:00000010:0.0:1775767931.570003:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220029400. 00020000:00000001:2.0:1775767931.570004:0:13471:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 02000000:00000001:0.0:1775767931.570009:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767931.570010:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00000100:00000040:1.0:1775767931.570011:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767931.570017:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:2.0:1775767931.570017:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00020000:00000001:2.0:1775767931.570021:0:13471:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767931.570022:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000040:1.0:1775767931.570023:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000010:0.0:1775767931.570029:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000100:00000001:1.0:1775767931.570030:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00020000:00000001:2.0:1775767931.570031:0:13471:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000020:00000040:1.0:1775767931.570038:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000020:00000001:2.0:1775767931.570039:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00000400:00000010:0.0:1775767931.570041:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000020:00000001:2.0:1775767931.570042:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000100:00080000:1.0:1775767931.570043:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00020000:00000001:2.0:1775767931.570050:0:13471:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00000100:00000001:1.0:1775767931.570051:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1775767931.570055:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294252800, portal 25 02000000:00000001:1.0:1775767931.570056:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00020000:00000001:2.0:1775767931.570057:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000001:0.0:1775767931.570062:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000020:00000001:2.0:1775767931.570062:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 02000000:00000001:1.0:1775767931.570068:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1775767931.570068:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000100:00000001:0.0:1775767931.570069:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606120192 : -112966103431424 : ffff994207888700) 00020000:00000001:2.0:1775767931.570075:0:13471:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767931.570077:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767931.570083:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767931.570084:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207888700 x1862022294252800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767956 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767931.570084:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00020000:00000001:2.0:1775767931.570092:0:13471:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000008:00100000:1.0:1775767931.570093:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000020:00000001:2.0:1775767931.570097:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000008:00000001:1.0:1775767931.570104:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767931.570106:0:13471:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000100:00000001:0.0:1775767931.570110:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:2.0:1775767931.570114:0:13471:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000100:00000001:1.0:1775767931.570115:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767931.570119:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:2.0:1775767931.570121:0:13471:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00000100:00000010:1.0:1775767931.570123:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207889f80. 00020000:00000001:2.0:1775767931.570123:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000040:0.0:1775767931.570130:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000020:00000001:2.0:1775767931.570130:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000040:1.0:1775767931.570132:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:2.0:1775767931.570134:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000400:00000010:0.0:1775767931.570138:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 02000000:00000001:1.0:1775767931.570141:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00020000:00000001:2.0:1775767931.570141:0:13471:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1775767931.570147:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294252800, offset 0 02000000:00000001:1.0:1775767931.570148:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767931.570148:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767931.570211:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00020000:00000001:2.0:1775767931.570212:0:13471:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 02000000:00000001:1.0:1775767931.570217:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000020:00000001:2.0:1775767931.570220:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000400:00000200:0.0:1775767931.570222:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000020:00000001:2.0:1775767931.570222:0:13471:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767931.570223:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767931.570236:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:2.0:1775767931.570237:0:13471:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:2.0:1775767931.570240:0:13471:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000400:00000200:0.0:1775767931.570241:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:2.0:1775767931.570243:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:2.0:1775767931.570245:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 02000000:00000010:1.0:1775767931.570246:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543f000. 00000020:00001000:2.0:1775767931.570248:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff994210b32090 00000400:00000200:0.0:1775767931.570252:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000020:00000010:2.0:1775767931.570255:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff9942060d4800. 00000100:00000001:1.0:1775767931.570257:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767931.570261:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000010:2.0:1775767931.570265:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942182d6cc0. 00000100:00100000:1.0:1775767931.570268:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207889f80 x1862022294252928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1775767931.570271:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000008:00000010:2.0:1775767931.570277:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207cc8b28. 00000400:00000200:0.0:1775767931.570282:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775767931.570291:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00020000:00000010:2.0:1775767931.570292:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994220156840. 00000100:00000001:1.0:1775767931.570293:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775767931.570300:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:2.0:1775767931.570301:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942077a9530. 00000400:00000200:0.0:1775767931.570305:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225148:225148:256:4294967295] 192.168.202.156@tcp LPNI seq info [225144:225144:8:4294967295] 00000080:00000010:2.0:1775767931.570310:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994208747738. 00000100:00000040:1.0:1775767931.570316:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ed2274f4] to pc [ptlrpcd_rcv+-1] req@ffff994207889f80 x1862022294252928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000010:2.0:1775767931.570322:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077ba910. 00000400:00000200:0.0:1775767931.570331:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000010:2.0:1775767931.570335:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9942060d5200. 00000008:00000010:2.0:1775767931.570343:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942077cd8c0. 00020000:00000010:2.0:1775767931.570355:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff99420876e400. 00000400:00000200:0.0:1775767931.570357:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00008000:00000010:2.0:1775767931.570364:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076c3f0. 00000100:00000001:1.0:1775767931.570369:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:2.0:1775767931.570371:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942050663c0. 00000100:00080000:1.0:1775767931.570376:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:0.0:1775767931.570378:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000020:00000010:2.0:1775767931.570383:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994209d2cc00. 00000100:00080000:1.0:1775767931.570386:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000010:0.0:1775767931.570387:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000020:00000010:2.0:1775767931.570388:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994210b32090. 00000100:00000001:1.0:1775767931.570389:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775767931.570395:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00020000:00000001:2.0:1775767931.570396:0:13471:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00000100:00000001:1.0:1775767931.570402:0:8:0:(import.c:484:import_select_connection()) Process entered 00020000:00000001:2.0:1775767931.570403:0:13471:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000800:00000200:0.0:1775767931.570404:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000080:00000001:2.0:1775767931.570407:0:13471:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000100:00080000:1.0:1775767931.570411:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5183 00000800:00000200:0.0:1775767931.570413:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000080:00200000:2.0:1775767931.570413:0:13471:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x10176:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000400:00000200:1.0:1775767931.570427:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000080:00000001:2.0:1775767931.570434:0:13471:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000800:00000200:0.1:1775767931.570438:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000001:2.0:1775767931.570439:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000400:00000200:1.0:1775767931.570446:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000020:00001000:2.0:1775767931.570446:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff994210b32630 00000800:00000200:0.1:1775767931.570447:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000100:00080000:1.0:1775767931.570456:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000020:00000010:2.0:1775767931.570456:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff9942060d4c00. 00000800:00000100:0.0:1775767931.570463:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767931.570468:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000020:00000010:2.0:1775767931.570468:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942182d6c00. 00000100:00000040:1.0:1775767931.570472:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000200:0.0:1775767931.570481:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:1.0:1775767931.570481:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000008:00000010:2.0:1775767931.570482:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207ccc2f0. 00000100:00000040:1.0:1775767931.570488:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00020000:00000010:2.0:1775767931.570491:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994220156e40. 00000400:00000200:0.0:1775767931.570492:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000100:00000001:1.0:1775767931.570498:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775767931.570503:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775767931.570505:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00008000:00000010:2.0:1775767931.570505:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942077a8e20. 00000400:00000200:0.0:1775767931.570506:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000020:00000001:1.0:1775767931.570506:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775767931.570511:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000400:00000010:0.0:1775767931.570512:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000080:00000010:2.0:1775767931.570514:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff9942087472a0. 00000020:00000001:1.0:1775767931.570515:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000001:0.0:1775767931.570519:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:1.0:1775767931.570520:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000080:00000010:2.0:1775767931.570529:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b8000. 00000100:00000040:1.0:1775767931.570530:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000200:0.0:1775767931.570531:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207888700 x1862022294252800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767956 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767931.570536:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000010:2.0:1775767931.570544:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9942060d5a00. 00000100:00000040:1.0:1775767931.570548:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000008:00000010:2.0:1775767931.570549:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942077cd600. 00020000:00000010:2.0:1775767931.570569:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff99420876d680. 00000100:00000001:1.0:1775767931.570570:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:0.0:1775767931.570574:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00008000:00000010:2.0:1775767931.570575:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076cf60. 00000020:00000040:1.0:1775767931.570578:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000080:00000010:2.0:1775767931.570583:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff994205066870. 00000100:00000040:0.0:1775767931.570585:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207888700 x1862022294252800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767956 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767931.570590:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000020:00000010:2.0:1775767931.570590:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9942060d5800. 00000020:00000010:2.0:1775767931.570602:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994210b32630. 00000100:00000001:1.0:1775767931.570604:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775767931.570611:0:13471:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1775767931.570614:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1775767931.570615:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775767931.570617:0:13471:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767931.570619:0:2399:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:1.0:1775767931.570621:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767931.570625:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767931.570627:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775767931.570629:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:1.0:1775767931.570631:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775767931.570637:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767931.570638:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775767931.570642:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299857726 00000100:00000001:1.0:1775767931.570642:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775767931.570650:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942078a3d80. 00000100:00000010:1.0:1775767931.570651:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff99420788b100. 00000020:00000040:1.0:1775767931.570655:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:1.0:1775767931.570657:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1775767931.570660:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 02000000:00000001:1.0:1775767931.570661:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767931.570665:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775767931.570667:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1775767931.570669:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299857726 02000000:00000001:1.0:1775767931.570669:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775767931.570673:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543d400. 00000100:00000010:0.0:1775767931.570674:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942078a34e0. 00000100:00000001:1.0:1775767931.570678:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767931.570679:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775767931.570684:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1775767931.570686:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff99420788b100 x1862022294253056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775767931.570687:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767931.570692:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775767931.570699:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767931/real 1775767931] req@ffff994207888700 x1862022294252800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767931.570717:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775767931.570721:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767931.570728:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e26cf977] to pc [ptlrpcd_rcv+-1] req@ffff99420788b100 x1862022294253056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775767931.570729:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775767931.570733:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775767931.570737:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000100:00000001:1.0:1775767931.570738:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767931.570743:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1775767931.570744:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000100:00080000:1.0:1775767931.570747:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000400:00000010:0.0:1775767931.570749:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:1.0:1775767931.570749:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775767931.570752:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775767931.570755:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1775767931.570755:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775767931.570759:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5183 00000400:00000200:1.0:1775767931.570764:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000200:0.0:1775767931.570765:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207888700 x1862022294252800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775767931.570768:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775767931.570771:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775767931.570778:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767931.570783:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:0.0:1775767931.570789:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207888700 x1862022294252800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775767931.570789:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767931.570796:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775767931.570802:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:0.0:1775767931.570803:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000040:1.0:1775767931.570805:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000100:00000001:0.0:1775767931.570806:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767931.570810:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:1.0:1775767931.570810:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:0.0:1775767931.570812:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:1.0:1775767931.570814:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000100:00100000:0.0:1775767931.570817:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207888700 x1862022294252800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1775767931.570820:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:1.0:1775767931.570826:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:0.0:1775767931.570830:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775767931.570834:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000040:0.0:1775767931.570837:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207888700 x1862022294252800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767956 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767931.570840:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767931.570845:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775767931.570850:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775767931.570851:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:0.0:1775767931.570855:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767931.570857:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000040:1.0:1775767931.570857:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00000001:0.0:1775767931.570860:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1775767931.570863:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775767931.570864:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775767931.570867:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775767931.570869:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775767931.570871:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:1.0:1775767931.570872:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00080000:0.0:1775767931.570876:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 02000000:00000001:1.0:1775767931.570877:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775767931.570880:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:1.0:1775767931.570882:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767931.570885:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207888700 x1862022294252800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767956 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767931.570887:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775767931.570891:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000100:00100000:0.0:1775767931.570895:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207888700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294252800:192.168.202.156@tcp:250:kworker.0 00000008:00000001:1.0:1775767931.570897:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767931.570903:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767931.570904:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767931.570907:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767931.570910:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207888700 x1862022294252800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767956 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1775767931.570910:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207888e00. 00000020:00000040:1.0:1775767931.570916:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00000100:00000001:0.0:1775767931.570920:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775767931.570922:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775767931.570923:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1775767931.570925:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775767931.570927:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220029400. 00000100:00000001:1.0:1775767931.570930:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775767931.570931:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775767931.570934:0:2399:0:(genops.c:1236:class_import_put()) Process entered 02000000:00000001:1.0:1775767931.570934:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000020:00000040:0.0:1775767931.570938:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 02000000:00000001:1.0:1775767931.570939:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767931.570940:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775767931.570944:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543c800. 02000000:00000010:1.0:1775767931.570946:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543fc00. 02000000:00000001:0.0:1775767931.570948:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775767931.570951:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1775767931.570951:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775767931.570954:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207888700. 00000100:00000001:0.0:1775767931.570958:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775767931.570959:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767931.570959:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207888e00 x1862022294253184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775767931.570965:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767931.570972:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775767931.570974:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767931.570974:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767931.570977:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767931.570980:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001812f966] to pc [ptlrpcd_rcv+-1] req@ffff994207888e00 x1862022294253184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767931.570981:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775767931.570984:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767931.570986:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775767931.570993:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207889f80 x1862022294252928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767931.570993:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767931.570998:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767931.571006:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775767931.571010:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775767931.571013:0:8:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775767931.571018:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:0.0:1775767931.571023:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:1.0:1775767931.571023:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5183 00000100:00000001:0.0:1775767931.571026:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767931.571029:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000200:1.0:1775767931.571030:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:0.0:1775767931.571031:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775767931.571035:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767931.571035:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775767931.571038:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775767931.571041:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207889f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294252928:192.168.202.156@tcp:38:kworker.0 00000100:00000040:1.0:1775767931.571044:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775767931.571047:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1775767931.571047:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1775767931.571050:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775767931.571052:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767931.571052:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000001:0.0:1775767931.571057:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1775767931.571057:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775767931.571061:0:8:0:(genops.c:811:class_conn2export()) Process entered 02000000:00000010:0.0:1775767931.571062:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220029400. 00000020:00000040:1.0:1775767931.571063:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 02000000:00000001:0.0:1775767931.571066:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775767931.571066:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775767931.571068:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:1.0:1775767931.571071:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000400:00000200:0.0:1775767931.571074:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:1.0:1775767931.571075:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000400:00000010:0.0:1775767931.571078:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000100:00000040:1.0:1775767931.571080:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767931.571082:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:0.0:1775767931.571084:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000040:1.0:1775767931.571084:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767931.571089:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000200:0.0:1775767931.571090:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294252928, portal 10 00000020:00000040:1.0:1775767931.571091:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00000001:0.0:1775767931.571093:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00080000:1.0:1775767931.571095:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775767931.571096:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606126464 : -112966103425152 : ffff994207889f80) 00000100:00000001:1.0:1775767931.571100:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767931.571102:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:0.0:1775767931.571105:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207889f80 x1862022294252928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767956 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1775767931.571105:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767931.571110:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767931.571114:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775767931.571117:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:1.0:1775767931.571122:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767931.571125:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1775767931.571125:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775767931.571128:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207889880. 00000100:00000040:0.0:1775767931.571132:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000020:00000040:1.0:1775767931.571132:0:8:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:1.0:1775767931.571135:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000010:0.0:1775767931.571138:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 02000000:00000001:1.0:1775767931.571139:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767931.571141:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775767931.571144:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000200:0.0:1775767931.571146:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294252928, offset 0 02000000:00000001:1.0:1775767931.571146:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775767931.571151:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543d000. 00000100:00000001:1.0:1775767931.571177:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767931.571183:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00100000:1.0:1775767931.571183:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207889880 x1862022294253312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1775767931.571193:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767931.571194:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775767931.571197:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767931.571200:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00000040:1.0:1775767931.571203:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f11856b8] to pc [ptlrpcd_rcv+-1] req@ffff994207889880 x1862022294253312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775767931.571205:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775767931.571211:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775767931.571216:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000100:00000001:1.0:1775767931.571219:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767931.571221:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00000040:1.0:1775767931.571224:0:8:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5194) 00000400:00000200:0.0:1775767931.571226:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225149:225149:256:4294967295] 192.168.202.156@tcp LPNI seq info [225145:225145:8:4294967295] 00000400:00000200:0.0:1775767931.571231:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767931.571236:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775767931.571240:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767931.571244:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775767931.571247:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775767931.571249:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775767931.571252:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767931.571253:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000100:0.0:1775767931.571256:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767931.571258:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775767931.571260:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775767931.571262:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775767931.571265:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775767931.571266:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775767931.571268:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775767931.571271:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207889f80 x1862022294252928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767956 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767931.571277:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767931.571279:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207889f80 x1862022294252928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767956 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767931.571284:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767931.571285:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775767931.571288:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767931.571291:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:0.0:1775767931.571293:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775767931.571295:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299856404 00000100:00000001:0.0:1775767931.571297:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299857726 00000100:00000001:0.0:1775767931.571299:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299856404 00000100:00000001:0.0:1775767931.571300:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299857726 00000100:00000001:0.0:1775767931.571302:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775767931.571304:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767931.571305:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775767931.571308:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767931/real 1775767931] req@ffff994207889f80 x1862022294252928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775767931.571313:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775767931.571314:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775767931.571315:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775767931.571316:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775767931.571317:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775767931.571319:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775767931.571321:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207889f80 x1862022294252928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775767931.571327:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207889f80 x1862022294252928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767931.571331:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775767931.571331:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767931.571333:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767931.571333:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775767931.571336:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207889f80 x1862022294252928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767931.571339:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775767931.571342:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207889f80 x1862022294252928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767956 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767931.571346:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775767931.571347:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767931.571348:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767931.571349:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767931.571350:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775767931.571352:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775767931.571354:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767931.571357:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775767931.571358:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775767931.571361:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207889f80 x1862022294252928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767956 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767931.571366:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207889f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294252928:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775767931.571368:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775767931.571370:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff99420788b100 x1862022294253056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767931.571374:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767931.571375:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767931.571376:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775767931.571378:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775767931.571379:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767931.571381:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff99420788b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294253056:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775767931.571383:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775767931.571385:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775767931.571386:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767931.571387:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775767931.571390:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220029800. 02000000:00000001:0.0:1775767931.571391:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767931.571393:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775767931.571395:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775767931.571397:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775767931.571399:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294253056, portal 10 00000100:00000001:0.0:1775767931.571400:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775767931.571401:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606130944 : -112966103420672 : ffff99420788b100) 00000100:00000040:0.0:1775767931.571406:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff99420788b100 x1862022294253056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767956 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767931.571410:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775767931.571412:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775767931.571414:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775767931.571415:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294253056, offset 0 00000400:00000200:0.0:1775767931.571418:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775767931.571420:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767931.571423:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775767931.571425:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775767931.571427:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775767931.571428:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775767931.571430:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775767931.571432:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225150:225150:256:4294967295] 192.168.202.156@tcp LPNI seq info [225146:225146:8:4294967295] 00000400:00000200:0.0:1775767931.571435:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767931.571440:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775767931.571442:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767931.571444:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775767931.571447:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775767931.571449:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775767931.571452:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767931.571453:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000100:0.0:1775767931.571456:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767931.571457:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775767931.571459:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775767931.571462:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775767931.571464:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775767931.571465:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775767931.571466:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775767931.571468:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff99420788b100 x1862022294253056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767956 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767931.571474:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767931.571476:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff99420788b100 x1862022294253056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767956 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767931.571480:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767931.571481:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775767931.571483:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767931.571485:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:0.0:1775767931.571488:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775767931.571489:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299856396 00000100:00000001:0.0:1775767931.571491:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299857726 00000100:00000001:0.0:1775767931.571492:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299856396 00000100:00000001:0.0:1775767931.571493:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299857726 00000100:00000001:0.0:1775767931.571495:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775767931.571496:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767931.571497:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775767931.571499:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767931/real 1775767931] req@ffff99420788b100 x1862022294253056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775767931.571503:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775767931.571504:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775767931.571505:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775767931.571506:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775767931.571507:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775767931.571509:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775767931.571511:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff99420788b100 x1862022294253056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775767931.571516:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff99420788b100 x1862022294253056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767931.571520:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775767931.571521:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767931.571522:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767931.571523:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775767931.571525:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff99420788b100 x1862022294253056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767931.571529:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775767931.571531:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff99420788b100 x1862022294253056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767956 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767931.571536:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775767931.571538:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767931.571539:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767931.571540:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767931.571541:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775767931.571543:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775767931.571545:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767931.571547:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775767931.571548:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775767931.571550:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff99420788b100 x1862022294253056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767956 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767931.571567:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff99420788b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294253056:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775767931.571570:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767931.571572:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767931.571574:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207889f80 x1862022294252928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767956 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767931.571579:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775767931.571580:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775767931.571581:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220029400. 02000000:00000001:0.0:1775767931.571583:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775767931.571584:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775767931.571585:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:0.0:1775767931.571587:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775767931.571588:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543f000. 02000000:00000001:0.0:1775767931.571590:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775767931.571591:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775767931.571592:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207889f80. 00000100:00000001:0.0:1775767931.571595:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775767931.571596:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767931.571597:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767931.571599:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff99420788b100 x1862022294253056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767956 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767931.571603:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775767931.571604:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775767931.571605:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220029800. 02000000:00000001:0.0:1775767931.571606:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775767931.571607:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775767931.571607:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:0.0:1775767931.571608:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775767931.571609:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543d400. 02000000:00000001:0.0:1775767931.571610:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775767931.571611:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775767931.571612:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff99420788b100. 00000100:00000001:0.0:1775767931.571613:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775767931.571614:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767931.571615:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767931.571617:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767931.571618:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767931.571620:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775767931.571621:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767931.571621:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775767931.571623:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207888e00 x1862022294253184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767931.571627:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767931.571628:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767931.571630:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775767931.571631:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775767931.571632:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767931.571634:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207888e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294253184:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775767931.571636:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775767931.571637:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775767931.571638:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767931.571640:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775767931.571642:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220029800. 02000000:00000001:0.0:1775767931.571644:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767931.571647:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775767931.571649:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775767931.571651:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775767931.571653:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294253184, portal 4 00000100:00000001:0.0:1775767931.571654:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775767931.571655:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606121984 : -112966103429632 : ffff994207888e00) 00000100:00000040:0.0:1775767931.571659:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207888e00 x1862022294253184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767956 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767931.571664:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775767931.571665:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775767931.571667:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775767931.571668:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294253184, offset 0 00000400:00000200:0.0:1775767931.571671:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775767931.571673:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767931.571675:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775767931.571677:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775767931.571679:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775767931.571681:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775767931.571683:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775767931.571685:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225151:225151:256:4294967295] 192.168.202.156@tcp LPNI seq info [225147:225147:8:4294967295] 00000400:00000200:0.0:1775767931.571687:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767931.571691:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775767931.571694:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767931.571696:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775767931.571698:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775767931.571700:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775767931.571703:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767931.571705:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000100:0.0:1775767931.571707:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767931.571709:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775767931.571711:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775767931.571714:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775767931.571717:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775767931.571719:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775767931.571722:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775767931.571725:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207888e00 x1862022294253184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767956 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767931.571733:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767931.571736:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207888e00 x1862022294253184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767956 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767931.571743:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767931.571744:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775767931.571748:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767931.571751:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:0.0:1775767931.571755:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775767931.571757:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299856419 00000100:00000001:0.0:1775767931.571760:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299857726 00000100:00000001:0.0:1775767931.571762:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299856419 00000100:00000001:0.0:1775767931.571765:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299857726 00000100:00000001:0.0:1775767931.571767:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775767931.571770:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767931.571771:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775767931.571776:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767931/real 1775767931] req@ffff994207888e00 x1862022294253184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775767931.571782:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775767931.571783:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775767931.571784:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775767931.571786:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775767931.571787:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775767931.571789:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775767931.571790:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207888e00 x1862022294253184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775767931.571801:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207888e00 x1862022294253184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767931.571805:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775767931.571806:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767931.571807:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767931.571808:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775767931.571810:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207888e00 x1862022294253184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767931.571814:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775767931.571816:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207888e00 x1862022294253184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767956 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767931.571820:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775767931.571821:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767931.571822:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767931.571823:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767931.571824:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775767931.571827:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775767931.571829:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767931.571831:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775767931.571833:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775767931.571835:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207888e00 x1862022294253184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767956 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767931.571841:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207888e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294253184:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775767931.571843:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775767931.571845:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207889880 x1862022294253312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767931.571853:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767931.571854:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767931.571856:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775767931.571857:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775767931.571858:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767931.571860:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207889880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294253312:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775767931.571862:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775767931.571863:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775767931.571864:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767931.571865:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775767931.571867:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220029400. 02000000:00000001:0.0:1775767931.571868:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767931.571870:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775767931.571872:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775767931.571874:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775767931.571876:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294253312, portal 4 00000100:00000001:0.0:1775767931.571877:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775767931.571878:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606124672 : -112966103426944 : ffff994207889880) 00000100:00000040:0.0:1775767931.571882:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207889880 x1862022294253312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767956 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767931.571887:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775767931.571888:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775767931.571889:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775767931.571891:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294253312, offset 0 00000400:00000200:0.0:1775767931.571893:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775767931.571895:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767931.571897:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775767931.571899:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775767931.571901:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775767931.571902:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775767931.571904:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775767931.571907:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225152:225152:256:4294967295] 192.168.202.156@tcp LPNI seq info [225148:225148:8:4294967295] 00000400:00000200:0.0:1775767931.571909:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767931.571914:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775767931.571917:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767931.571919:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775767931.571922:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775767931.571924:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775767931.571927:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767931.571929:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000100:0.0:1775767931.571932:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767931.571933:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775767931.571935:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775767931.571937:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775767931.571939:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775767931.571940:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775767931.571941:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775767931.571943:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207889880 x1862022294253312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767956 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767931.571949:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767931.571951:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207889880 x1862022294253312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767956 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767931.571955:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767931.571956:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775767931.571957:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767931.571959:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:0.0:1775767931.571961:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775767931.571962:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299856422 00000100:00000001:0.0:1775767931.571963:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299857726 00000100:00000001:0.0:1775767931.571965:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299856422 00000100:00000001:0.0:1775767931.571967:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299857726 00000100:00000001:0.0:1775767931.571968:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775767931.571970:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767931.571971:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775767931.571973:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767931/real 1775767931] req@ffff994207889880 x1862022294253312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775767931.571979:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775767931.571979:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775767931.571980:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775767931.571981:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775767931.571982:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775767931.571983:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775767931.571985:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207889880 x1862022294253312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775767931.571990:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207889880 x1862022294253312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767931.571994:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775767931.571995:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767931.571997:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767931.571998:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775767931.572000:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207889880 x1862022294253312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767931.572004:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775767931.572006:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207889880 x1862022294253312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767956 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767931.572010:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775767931.572011:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767931.572012:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767931.572013:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767931.572014:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775767931.572016:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775767931.572018:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767931.572020:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775767931.572022:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775767931.572024:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207889880 x1862022294253312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767956 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767931.572029:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207889880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294253312:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775767931.572031:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767931.572032:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767931.572034:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207888e00 x1862022294253184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767956 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767931.572038:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775767931.572039:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775767931.572040:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220029800. 02000000:00000001:0.0:1775767931.572041:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775767931.572042:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775767931.572043:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:0.0:1775767931.572044:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775767931.572045:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543fc00. 02000000:00000001:0.0:1775767931.572046:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775767931.572047:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775767931.572048:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207888e00. 00000100:00000001:0.0:1775767931.572050:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775767931.572051:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767931.572052:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767931.572054:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207889880 x1862022294253312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767956 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767931.572059:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775767931.572059:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775767931.572060:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220029400. 02000000:00000001:0.0:1775767931.572061:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775767931.572062:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775767931.572062:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:0.0:1775767931.572063:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775767931.572065:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543d000. 02000000:00000001:0.0:1775767931.572066:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775767931.572067:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775767931.572068:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207889880. 00000100:00000001:0.0:1775767931.572070:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775767931.572071:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767931.572072:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767931.572074:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767931.572075:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767931.572076:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775767931.572077:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767931.633266:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767931.633273:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767931.633278:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767931.633279:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767931.633283:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767931.633284:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767931.633286:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767931.633287:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767931.633289:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767931.633290:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767931.633309:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767931.633311:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767931.633314:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767931.697328:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767931.697336:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767931.697337:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767931.697343:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767931.697347:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767931.697350:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767931.697353:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767931.697355:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767931.697357:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767932.593222:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767932.593227:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767932.593231:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767932.593237:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775767932.593240:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767932.593241:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767932.593246:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767932.593249:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767932.593249:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767932.593254:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767932.593256:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775767932.593265:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775767932.593271:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767932.593272:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767932.593274:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767932.593282:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767932.593283:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767932.593289:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767932.593290:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767932.593294:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767932.593295:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:0.0:1775767932.593300:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767932.593301:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767932.593306:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767932.593309:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767932.593311:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767932.593314:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767932.657285:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767932.657292:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767932.657298:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767932.657299:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767932.657303:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767932.657304:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767932.657307:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767932.657309:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767932.657312:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767932.657313:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767932.657338:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767932.657341:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767932.657345:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767932.721278:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767932.721289:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767932.721292:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767932.721299:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767932.721306:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767932.721311:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767932.721314:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767932.721318:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767932.721322:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767932.913270:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299857726, at 4299857728 00000100:00000040:0.0:1775767932.913288:0:14953:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4299857726, at 4299857728 00000100:00000010:1.0:1775767932.913288:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a34e0. 00000100:00000010:0.0:1775767932.913301:0:14953:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3d80. 00000100:00000001:0.0:1775767933.617217:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767933.617223:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767933.617226:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767933.617232:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775767933.617236:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767933.617240:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767933.617243:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767933.617246:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767933.617249:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767933.617287:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767933.617293:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767933.617296:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767933.617301:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767933.617305:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767933.617309:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767933.617311:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767933.617314:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767933.617317:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767933.618193:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767933.618196:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767933.618198:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767933.618200:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767933.618202:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767933.618204:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767933.618206:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767933.618207:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767933.618209:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767933.681249:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767933.681253:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767933.681257:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767933.681258:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767933.681261:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767933.681262:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767933.681265:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767933.681266:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767933.681268:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767933.681269:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767933.681283:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767933.681285:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767933.681287:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767933.745243:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767933.745247:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767933.745249:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767933.745254:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767933.745257:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767933.745260:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767933.745262:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767933.745264:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767933.745267:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767934.641227:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767934.641234:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767934.641236:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767934.641241:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767934.641248:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767934.641251:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767934.641253:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767934.641255:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767934.641257:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767934.641286:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767934.641289:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767934.641291:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767934.641293:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767934.641295:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767934.641297:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767934.641299:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767934.641300:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767934.641302:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767934.641332:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767934.641336:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767934.641337:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767934.641341:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775767934.641344:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767934.641351:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767934.641354:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767934.641356:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767934.641357:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767934.705280:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767934.705286:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767934.705290:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767934.705291:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767934.705293:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767934.705294:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767934.705296:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767934.705297:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767934.705300:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767934.705301:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767934.705327:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767934.705329:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767934.705331:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767934.769269:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767934.769276:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767934.769278:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767934.769284:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767934.769288:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767934.769291:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767934.769293:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767934.769295:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767934.769297:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767935.665215:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767935.665222:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767935.665224:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767935.665224:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767935.665228:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:3.0:1775767935.665230:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767935.665231:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767935.665236:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767935.665236:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767935.665240:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767935.665241:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767935.665245:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767935.665245:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767935.665249:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767935.665252:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767935.665255:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767935.665258:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767935.665260:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767935.665277:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767935.665281:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767935.665282:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767935.665284:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767935.665286:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767935.665288:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767935.665289:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767935.665290:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767935.665292:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767935.729245:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767935.729250:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767935.729254:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767935.729257:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775767935.729259:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775767935.729261:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767935.729264:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775767935.729265:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767935.729267:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767935.729269:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767935.729271:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767935.729272:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775767935.729273:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775767935.729274:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767935.729275:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775767935.729275:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767935.729277:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767935.729278:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767935.729280:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767935.729282:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775767935.729283:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775767935.729284:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767935.729285:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775767935.729285:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767935.729286:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767935.729288:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767935.729290:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767935.729291:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775767935.729291:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775767935.729292:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767935.729293:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775767935.729294:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767935.729295:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767935.729297:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767935.729312:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767935.729314:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767935.729315:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767935.793247:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767935.793252:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767935.793254:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767935.793259:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767935.793262:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767935.793265:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767935.793267:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767935.793269:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767935.793271:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767936.689250:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767936.689250:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767936.689261:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767936.689262:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767936.689265:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767936.689266:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767936.689274:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775767936.689276:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767936.689283:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767936.689284:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767936.689288:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767936.689289:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767936.689291:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767936.689291:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767936.689300:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767936.689300:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767936.689307:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767936.689308:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767936.689309:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767936.689315:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767936.689320:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775767936.689322:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767936.689326:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767936.689330:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767936.689334:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767936.689336:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767936.689340:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1775767936.689347:0:8:0:(rw.c:1481:ll_writepages()) Process entered 00000080:00000001:0.0:1775767936.689353:0:8:0:(file.c:5250:cl_sync_file_range()) Process entered 00000100:00080000:3.0:1775767936.689355:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000001:0.0:1775767936.689357:0:8:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00080000:3.0:1775767936.689363:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775767936.689366:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767936.689369:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000020:00000010:0.0:1775767936.689370:0:8:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff9942103bed80. 00000100:00000001:3.0:1775767936.689379:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000020:00000010:0.0:1775767936.689382:0:8:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0fe00. 00000100:00080000:3.0:1775767936.689389:0:13471:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5189 00000020:00000010:0.0:1775767936.689394:0:8:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210b0f800. 00000400:00000200:3.0:1775767936.689399:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000010:0.0:1775767936.689400:0:8:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942078a3d80. 00000400:00000200:3.0:1775767936.689407:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000008:00000010:0.0:1775767936.689411:0:8:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff9942086e6468. 00000100:00080000:3.0:1775767936.689411:0:13471:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767936.689418:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00020000:00000010:0.0:1775767936.689419:0:8:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994209592f00. 00000100:00000001:3.0:1775767936.689424:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767936.689427:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00008000:00000010:0.0:1775767936.689428:0:8:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff99422035b880. 00000100:00000001:3.0:1775767936.689435:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000080:00000010:0.0:1775767936.689438:0:8:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994207e3e150. 00000020:00000001:3.0:1775767936.689439:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767936.689443:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000080:00000010:0.0:1775767936.689446:0:8:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b2238. 00000020:00000001:3.0:1775767936.689447:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000010:0.0:1775767936.689453:0:8:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0f200. 00000020:00000040:3.0:1775767936.689453:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:3.0:1775767936.689462:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000008:00000010:0.0:1775767936.689464:0:8:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff99420906be40. 00000020:00000001:3.0:1775767936.689466:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00020000:00000010:0.0:1775767936.689469:0:8:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff9942103ccfc0. 00000100:00000040:3.0:1775767936.689472:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767936.689477:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00008000:00000010:0.0:1775767936.689478:0:8:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076afa0. 00000100:00000040:3.0:1775767936.689483:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000080:00000010:0.0:1775767936.689484:0:8:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942107690f0. 00000100:00000001:3.0:1775767936.689492:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775767936.689494:0:8:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107752095112 : -112965957456504 : ffff9942103bed88) 00000020:00000040:3.0:1775767936.689497:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000020:00001000:0.0:1775767936.689504:0:8:0:(cl_object.c:831:cl_env_get()) 1@ffff9942103bed80 00000100:00080000:3.0:1775767936.689521:0:13471:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000020:00000001:0.0:1775767936.689524:0:8:0:(cl_object.c:151:cl_object_top()) ffff9942078a76c0 -> ffff9942078a76c0 00000100:00000001:3.0:1775767936.689525:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767936.689530:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000020:00000001:0.0:1775767936.689532:0:8:0:(cl_io.c:125:__cl_io_init()) Process entered 02000000:00000001:3.0:1775767936.689534:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767936.689538:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1775767936.689539:0:8:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000100:00000001:3.0:1775767936.689540:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000080:00200000:0.0:1775767936.689545:0:8:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000100:00000001:3.0:1775767936.689546:0:13471:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767936.689557:0:8:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000100:00000010:3.0:1775767936.689558:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8a80. 00000020:00000001:0.0:1775767936.689563:0:8:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000040:3.0:1775767936.689563:0:13471:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 00000080:00000001:0.0:1775767936.689571:0:8:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767936.689572:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767936.689574:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767936.689581:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00020000:00000002:0.0:1775767936.689583:0:8:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017a:0x0]io ffff9942077b2298 type 6 ignore/verify layout 1/0 02000000:00000001:3.0:1775767936.689584:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00020000:00000002:0.0:1775767936.689589:0:8:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207c1f9b0) owner 0000000000000000/ffff994201224f80 02000000:00000001:3.0:1775767936.689590:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767936.689600:0:8:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 02000000:00000010:3.0:1775767936.689602:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2c00. 00020000:00000001:0.0:1775767936.689603:0:8:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1775767936.689611:0:8:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00020000:00000001:0.0:1775767936.689613:0:8:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767936.689614:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767936.689616:0:8:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1775767936.689624:0:8:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1775767936.689626:0:8:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767936.689627:0:8:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1775767936.689632:0:8:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00100000:3.0:1775767936.689633:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8a80 x1862022294253440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00020000:00000001:0.0:1775767936.689637:0:8:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00020000:00000002:0.0:1775767936.689642:0:8:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207c1f9b0) owner 0000000000000000/ffff994201224f80 00000020:00000001:0.0:1775767936.689652:0:8:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767936.689661:0:8:0:(cl_io.c:823:cl_io_loop()) Process entered 00000100:00000001:3.0:1775767936.689663:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000020:00000001:0.0:1775767936.689664:0:8:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00020000:00000001:0.0:1775767936.689668:0:8:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00020000:00200000:0.0:1775767936.689672:0:8:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00000001:3.0:1775767936.689673:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1775767936.689677:0:8:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:0.0:1775767936.689680:0:8:0:(lov_io.c:155:lov_sub_get()) Process entered 00020000:00000001:0.0:1775767936.689685:0:8:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000100:00000040:3.0:1775767936.689686:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006c3265af] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8a80 x1862022294253440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775767936.689687:0:8:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:0.0:1775767936.689690:0:8:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff9942103be000. 00000020:00000010:0.0:1775767936.689698:0:8:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0f600. 00000020:00000010:0.0:1775767936.689708:0:8:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210b0e000. 00000020:00000010:0.0:1775767936.689716:0:8:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942078a3a20. 00000008:00000010:0.0:1775767936.689724:0:8:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff9942086e4e18. 00020000:00000010:0.0:1775767936.689726:0:8:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994209592e40. 00008000:00000010:0.0:1775767936.689729:0:8:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994220358000. 00000080:00000010:0.0:1775767936.689732:0:8:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994207e3e3f0. 00000080:00000010:0.0:1775767936.689734:0:8:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b0ff8. 00000020:00000010:0.0:1775767936.689736:0:8:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0ec00. 00000100:00000001:3.0:1775767936.689738:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:0.0:1775767936.689739:0:8:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff99420906a6e0. 00020000:00000010:0.0:1775767936.689749:0:8:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff9942103cc480. 00000100:00080000:3.0:1775767936.689749:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00008000:00000010:0.0:1775767936.689756:0:8:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076adb0. 00000100:00080000:3.0:1775767936.689759:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775767936.689762:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000080:00000010:0.0:1775767936.689763:0:8:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff994210769870. 00000100:00080000:3.0:1775767936.689765:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000020:00000001:0.0:1775767936.689774:0:8:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107752091656 : -112965957459960 : ffff9942103be008) 00000100:00000001:3.0:1775767936.689774:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775767936.689776:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00001000:0.0:1775767936.689786:0:8:0:(cl_object.c:831:cl_env_get()) 1@ffff9942103be000 00000100:00000001:1.0:1775767936.689786:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1775767936.689789:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5189 00000100:00000001:1.0:1775767936.689792:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000001:0.0:1775767936.689793:0:8:0:(cl_object.c:151:cl_object_top()) ffff9942078a7788 -> ffff9942078a76c0 00000020:00000001:0.0:1775767936.689802:0:8:0:(cl_io.c:125:__cl_io_init()) Process entered 00000400:00000200:3.0:1775767936.689802:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000040:1.0:1775767936.689808:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a8a80 x1862022294253440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775767936.689811:0:8:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000400:00000200:3.0:1775767936.689813:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000020:00000001:0.0:1775767936.689817:0:8:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00080000:3.0:1775767936.689820:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000020:00000001:0.0:1775767936.689825:0:8:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767936.689831:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00020000:00000001:0.0:1775767936.689832:0:8:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767936.689841:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00020000:00000001:0.0:1775767936.689844:0:8:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107752153128 : -112965957398488 : ffff9942103cd028) 00000100:00000001:1.0:1775767936.689845:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000040:3.0:1775767936.689850:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767936.689851:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767936.689853:0:8:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000020:00000001:0.0:1775767936.689860:0:8:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767936.689863:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00020000:00000001:0.0:1775767936.689865:0:8:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767936.689865:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:3.0:1775767936.689866:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767936.689873:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:0.0:1775767936.689874:0:8:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767936.689874:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1775767936.689876:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767936.689882:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 02000000:00000001:1.0:1775767936.689883:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767936.689884:0:8:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:3.0:1775767936.689890:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00020000:00000001:0.0:1775767936.689891:0:8:0:(lov_io.c:1108:lov_io_lock()) Process entered 00000100:00100000:1.0:1775767936.689899:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294253440:192.168.202.156@tcp:250:kworker.0 00000020:00000001:3.0:1775767936.689899:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00020000:00000001:0.0:1775767936.689900:0:8:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:0.0:1775767936.689905:0:8:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:0.0:1775767936.689911:0:8:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:0.0:1775767936.689914:0:8:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767936.689914:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000040:3.0:1775767936.689914:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 02000000:00000001:1.0:1775767936.689921:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:0.0:1775767936.689922:0:8:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767936.689923:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1775767936.689925:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767936.689927:0:8:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767936.689935:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00020000:00000001:0.0:1775767936.689936:0:8:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767936.689936:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000010:1.0:1775767936.689945:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543dc00. 00000020:00000001:0.0:1775767936.689947:0:8:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000100:00000001:3.0:1775767936.689947:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775767936.689952:0:8:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767936.689952:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775767936.689953:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000020:00000001:0.0:1775767936.689962:0:8:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767936.689965:0:8:0:(cl_io.c:528:cl_io_start()) Process entered 00000100:00080000:3.0:1775767936.689967:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00020000:00000001:0.0:1775767936.689970:0:8:0:(lov_io.c:1114:lov_io_start()) Process entered 00000400:00000200:1.0:1775767936.689970:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:0.0:1775767936.689976:0:8:0:(lov_io.c:1086:lov_io_call()) Process entered 00000400:00000010:1.0:1775767936.689980:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023cc08. 00000020:00000001:0.0:1775767936.689983:0:8:0:(cl_io.c:528:cl_io_start()) Process entered 00000100:00000001:3.0:1775767936.689983:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1775767936.689987:0:8:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 02000000:00000001:3.0:1775767936.689988:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:1.0:1775767936.689989:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf67f8. 00000008:00000001:0.0:1775767936.689994:0:8:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 02000000:00000001:3.0:1775767936.689997:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000200:1.0:1775767936.690001:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294253440, portal 25 02000000:00000001:3.0:1775767936.690001:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1775767936.690002:0:8:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:0.0:1775767936.690008:0:8:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000100:00000001:3.0:1775767936.690013:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00000001:0.0:1775767936.690014:0:8:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767936.690015:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775767936.690021:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640855168 : -112966068696448 : ffff9942099a8a80) 00000008:00100000:3.0:1775767936.690022:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775767936.690023:0:8:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000008:00000001:0.0:1775767936.690026:0:8:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000008:00000001:0.0:1775767936.690033:0:8:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107884691392 : -112965824860224 : ffff994218232fc0) 00000008:00000001:3.0:1775767936.690033:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767936.690044:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a8a80 x1862022294253440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767961 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000020:0.0:1775767936.690045:0:8:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994218232fc0 ready 0|-|- wr 1|-|+ rd 0|- 10 in flight 00000100:00000001:3.0:1775767936.690045:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767936.690056:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aa300. 00000008:00000001:0.0:1775767936.690059:0:8:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:0.0:1775767936.690063:0:8:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:0.0:1775767936.690067:0:8:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1775767936.690067:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000008:00000001:0.0:1775767936.690078:0:8:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 02000000:00000001:3.0:1775767936.690079:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000008:00000020:0.0:1775767936.690081:0:8:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994210675880 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 02000000:00000001:3.0:1775767936.690083:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767936.690085:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1775767936.690093:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000008:00000001:0.0:1775767936.690095:0:8:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1775767936.690098:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:1.0:1775767936.690102:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000008:00000001:0.0:1775767936.690103:0:8:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767936.690104:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775767936.690110:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000020:00000001:0.0:1775767936.690112:0:8:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767936.690116:0:8:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767936.690117:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b3400. 00020000:00000001:0.0:1775767936.690118:0:8:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775767936.690119:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294253440, offset 0 00000020:00000001:0.0:1775767936.690123:0:8:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767936.690127:0:8:0:(cl_io.c:555:cl_io_end()) Process entered 00000100:00000001:3.0:1775767936.690127:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767936.690129:0:8:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00020000:00000001:0.0:1775767936.690133:0:8:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000400:00000200:1.0:1775767936.690134:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000020:00000001:0.0:1775767936.690136:0:8:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000001:0.0:1775767936.690139:0:8:0:(cl_io.c:563:cl_io_end()) Process leaving 00020000:00000001:0.0:1775767936.690141:0:8:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767936.690146:0:8:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000100:00100000:3.0:1775767936.690147:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aa300 x1862022294253568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000001:0.0:1775767936.690149:0:8:0:(cl_io.c:563:cl_io_end()) Process leaving 00000400:00000200:1.0:1775767936.690149:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:0.0:1775767936.690153:0:8:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000400:00000200:1.0:1775767936.690198:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00020000:00000001:0.0:1775767936.690206:0:8:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00000400:00000200:1.0:1775767936.690209:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00020000:00000001:0.0:1775767936.690211:0:8:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:0.0:1775767936.690216:0:8:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000020:00000001:0.0:1775767936.690218:0:8:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000400:00000200:1.0:1775767936.690221:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00020000:00000001:0.0:1775767936.690223:0:8:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767936.690231:0:8:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767936.690232:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1775767936.690233:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00020000:00000001:0.0:1775767936.690235:0:8:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000020:00000001:0.0:1775767936.690240:0:8:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000400:00000200:1.0:1775767936.690243:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000020:00000001:0.0:1775767936.690244:0:8:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000100:00000001:3.0:1775767936.690244:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767936.690247:0:8:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000001:0.0:1775767936.690249:0:8:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:0.0:1775767936.690253:0:8:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00000400:00000200:1.0:1775767936.690253:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225153:225153:256:4294967295] 192.168.202.156@tcp LPNI seq info [225149:225149:8:4294967295] 00020000:00000001:0.0:1775767936.690257:0:8:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:0.0:1775767936.690261:0:8:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000001:0.0:1775767936.690262:0:8:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000100:00000040:3.0:1775767936.690262:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f33bdf15] to pc [ptlrpcd_rcv+-1] req@ffff9942099aa300 x1862022294253568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:0.0:1775767936.690266:0:8:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767936.690266:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:0.0:1775767936.690271:0:8:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767936.690282:0:8:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000020:00000001:0.0:1775767936.690288:0:8:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000020:00000001:0.0:1775767936.690294:0:8:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767936.690302:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000020:00000001:0.0:1775767936.690304:0:8:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1775767936.690310:0:8:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:0.0:1775767936.690316:0:8:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:0.0:1775767936.690321:0:8:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:0.0:1775767936.690326:0:8:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1775767936.690330:0:8:0:(cl_object.c:925:cl_env_put()) 1@ffff9942103be000 00000800:00000200:1.0:1775767936.690334:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00000001:3.0:1775767936.690334:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1775767936.690336:0:8:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210b0e000. 00000020:00000010:0.0:1775767936.690348:0:8:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942078a3a20. 00000100:00080000:3.0:1775767936.690348:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000010:1.0:1775767936.690349:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541200. 00000008:00000010:0.0:1775767936.690357:0:8:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff9942086e4e18. 00000100:00080000:3.0:1775767936.690361:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775767936.690363:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00020000:00000010:0.0:1775767936.690364:0:8:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994209592e40. 00000100:00080000:3.0:1775767936.690370:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:1.0:1775767936.690371:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00008000:00000010:0.0:1775767936.690372:0:8:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994220358000. 00000100:00000001:3.0:1775767936.690373:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000080:00000010:0.0:1775767936.690383:0:8:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994207e3e3f0. 00000800:00000200:1.0:1775767936.690383:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000100:00080000:3.0:1775767936.690384:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5189 00000080:00000010:0.0:1775767936.690392:0:8:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b0ff8. 00000400:00000200:3.0:1775767936.690397:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000010:0.0:1775767936.690400:0:8:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0f600. 00000400:00000200:3.0:1775767936.690405:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000800:00000200:1.1:1775767936.690406:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000008:00000010:0.0:1775767936.690407:0:8:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff99420906a6e0. 00000100:00080000:3.0:1775767936.690411:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00020000:00000010:0.0:1775767936.690416:0:8:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff9942103cc480. 00000800:00000200:1.1:1775767936.690420:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000100:00000040:3.0:1775767936.690423:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00008000:00000010:0.0:1775767936.690425:0:8:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076adb0. 00000100:00000001:3.0:1775767936.690435:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000080:00000010:0.0:1775767936.690438:0:8:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff994210769870. 00000020:00000010:0.0:1775767936.690444:0:8:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0ec00. 00000100:00000040:3.0:1775767936.690449:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000010:0.0:1775767936.690451:0:8:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff9942103be000. 00000800:00000100:1.0:1775767936.690452:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000100:00000001:3.0:1775767936.690460:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000800:00000010:1.0:1775767936.690461:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541200. 00020000:00000001:0.0:1775767936.690462:0:8:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:0.0:1775767936.690466:0:8:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000020:00000001:3.0:1775767936.690468:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000080:00000001:0.0:1775767936.690470:0:8:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000020:00000040:3.0:1775767936.690471:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000080:00200000:0.0:1775767936.690474:0:8:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000400:00000200:1.0:1775767936.690474:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000001:3.0:1775767936.690479:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000080:00000001:0.0:1775767936.690484:0:8:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000040:3.0:1775767936.690485:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:0.0:1775767936.690487:0:8:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1775767936.690492:0:8:0:(cl_object.c:925:cl_env_put()) 1@ffff9942103bed80 00000400:00000200:1.0:1775767936.690494:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000020:00000001:3.0:1775767936.690495:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000010:0.0:1775767936.690496:0:8:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210b0f800. 00000020:00000010:0.0:1775767936.690518:0:8:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942078a3d80. 00000020:00000001:3.0:1775767936.690520:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000008:00000010:0.0:1775767936.690524:0:8:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff9942086e6468. 00000400:00000200:1.0:1775767936.690530:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000100:00000040:3.0:1775767936.690531:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00020000:00000010:0.0:1775767936.690533:0:8:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994209592f00. 00000400:00000010:1.0:1775767936.690536:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000100:00000001:3.0:1775767936.690548:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00008000:00000010:0.0:1775767936.690551:0:8:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff99422035b880. 00000100:00000001:1.0:1775767936.690551:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:3.0:1775767936.690555:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000080:00000010:0.0:1775767936.690559:0:8:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994207e3e150. 00000100:00000200:1.0:1775767936.690562:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a8a80 x1862022294253440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767961 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767936.690564:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000080:00000010:0.0:1775767936.690566:0:8:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b2238. 00000020:00000010:0.0:1775767936.690575:0:8:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0fe00. 00000020:00000040:3.0:1775767936.690576:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000008:00000010:0.0:1775767936.690582:0:8:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff99420906be40. 00020000:00000010:0.0:1775767936.690588:0:8:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff9942103ccfc0. 00000100:00080000:3.0:1775767936.690593:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00008000:00000010:0.0:1775767936.690596:0:8:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076afa0. 00000100:00000001:1.0:1775767936.690599:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1775767936.690601:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1775767936.690602:0:8:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942107690f0. 00000020:00000010:0.0:1775767936.690609:0:8:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0f200. 00000100:00000040:1.0:1775767936.690612:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a8a80 x1862022294253440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767961 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1775767936.690612:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000020:00000010:0.0:1775767936.690613:0:8:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff9942103bed80. 00000080:00000001:0.0:1775767936.690618:0:8:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1775767936.690619:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000001:0.0:1775767936.690627:0:8:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767936.690631:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1775767936.690639:0:8:0:(rw.c:1481:ll_writepages()) Process entered 00000100:00000001:3.0:1775767936.690640:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000080:00000001:0.0:1775767936.690646:0:8:0:(file.c:5250:cl_sync_file_range()) Process entered 00000100:00000001:1.0:1775767936.690646:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1775767936.690647:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000020:00000001:0.0:1775767936.690649:0:8:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00000001:1.0:1775767936.690654:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000010:0.0:1775767936.690655:0:8:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff9942103bed80. 00000008:00000001:3.0:1775767936.690656:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1775767936.690662:0:8:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0f200. 00000100:00000001:3.0:1775767936.690664:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1775767936.690666:0:8:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210b0fe00. 00000020:00000010:0.0:1775767936.690672:0:8:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942078a3d80. 00000100:00000001:1.0:1775767936.690674:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:0.0:1775767936.690675:0:8:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff9942086e6468. 00000100:00000010:3.0:1775767936.690676:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ab800. 00020000:00000010:0.0:1775767936.690681:0:8:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994209592f00. 00008000:00000010:0.0:1775767936.690686:0:8:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff99422035b880. 00000020:00000040:3.0:1775767936.690687:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000080:00000010:0.0:1775767936.690693:0:8:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994207e3e150. 00000400:00000200:1.0:1775767936.690693:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:3.0:1775767936.690695:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000080:00000010:0.0:1775767936.690697:0:8:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b2238. 02000000:00000001:3.0:1775767936.690702:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1775767936.690704:0:8:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0f800. 00000100:00000001:1.0:1775767936.690704:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299862846 00000008:00000010:0.0:1775767936.690711:0:8:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff99420906be40. 00000100:00000001:3.0:1775767936.690713:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000010:1.0:1775767936.690715:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba600. 02000000:00000001:3.0:1775767936.690716:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00020000:00000010:0.0:1775767936.690717:0:8:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff9942103ccfc0. 00008000:00000010:0.0:1775767936.690723:0:8:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076afa0. 02000000:00000001:3.0:1775767936.690724:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1775767936.690729:0:8:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942107690f0. 00000100:00000001:1.0:1775767936.690729:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 02000000:00000010:3.0:1775767936.690732:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b1000. 00000020:00000001:0.0:1775767936.690734:0:8:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107752095112 : -112965957456504 : ffff9942103bed88) 00000020:00001000:0.0:1775767936.690738:0:8:0:(cl_object.c:831:cl_env_get()) 1@ffff9942103bed80 00000100:00000001:3.0:1775767936.690738:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767936.690740:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299862846 00000020:00000001:0.0:1775767936.690749:0:8:0:(cl_object.c:151:cl_object_top()) ffff9942078a7f58 -> ffff9942078a7f58 00000100:00000010:1.0:1775767936.690750:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba9c0. 00000020:00000001:0.0:1775767936.690754:0:8:0:(cl_io.c:125:__cl_io_init()) Process entered 00000100:00100000:3.0:1775767936.690756:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ab800 x1862022294253696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000080:00000001:0.0:1775767936.690758:0:8:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000100:00000001:1.0:1775767936.690758:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775767936.690765:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00200000:0.0:1775767936.690767:0:8:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017b:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000100:00000001:1.0:1775767936.690776:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767936.690787:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000020:00000001:0.0:1775767936.690791:0:8:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1775767936.690798:0:8:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00100000:1.0:1775767936.690803:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767936/real 1775767936] req@ffff9942099a8a80 x1862022294253440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000001:0.0:1775767936.690805:0:8:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767936.690805:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767936.690816:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1775767936.690817:0:8:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017b:0x0]io ffff9942077b2298 type 6 ignore/verify layout 1/0 00020000:00000002:0.0:1775767936.690834:0:8:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207c1f000) owner 0000000000000000/ffff994201224f80 00000400:00000001:1.0:1775767936.690836:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:3.0:1775767936.690842:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000089df80c0] to pc [ptlrpcd_rcv+-1] req@ffff9942099ab800 x1862022294253696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:0.0:1775767936.690844:0:8:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00000400:00000001:1.0:1775767936.690844:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00020000:00000001:0.0:1775767936.690851:0:8:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00000400:00000010:1.0:1775767936.690851:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023cc08. 00020000:00000001:0.0:1775767936.690858:0:8:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00000400:00000200:1.0:1775767936.690862:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf67f8 00020000:00000001:0.0:1775767936.690865:0:8:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775767936.690870:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf67f8. 00020000:00000001:0.0:1775767936.690878:0:8:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775767936.690880:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1775767936.690888:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767936.690889:0:8:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00000100:00000200:1.0:1775767936.690889:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a8a80 x1862022294253440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:0.0:1775767936.690894:0:8:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767936.690894:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000001:0.0:1775767936.690904:0:8:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1775767936.690909:0:8:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00080000:3.0:1775767936.690914:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00020000:00000001:0.0:1775767936.690915:0:8:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00020000:00000002:0.0:1775767936.690923:0:8:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207c1f000) owner 0000000000000000/ffff994201224f80 00000100:00000001:3.0:1775767936.690924:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767936.690931:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000020:00000001:0.0:1775767936.690932:0:8:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775767936.690932:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a8a80 x1862022294253440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775767936.690942:0:8:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:0.0:1775767936.690947:0:8:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000100:00000001:3.0:1775767936.690948:0:13471:0:(import.c:484:import_select_connection()) Process entered 00020000:00000001:0.0:1775767936.690957:0:8:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00000100:00000001:1.0:1775767936.690958:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00020000:00200000:0.0:1775767936.690963:0:8:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00080000:3.0:1775767936.690963:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5189 00000100:00000001:1.0:1775767936.690964:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00020000:00000002:0.0:1775767936.690974:0:8:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00000100:00000001:1.0:1775767936.690975:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767936.690981:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1775767936.690985:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:0.0:1775767936.690986:0:8:0:(lov_io.c:155:lov_sub_get()) Process entered 00020000:00000001:0.0:1775767936.690993:0:8:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000100:00100000:1.0:1775767936.690993:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a8a80 x1862022294253440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775767936.690997:0:8:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000400:00000200:3.0:1775767936.690999:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000020:00000010:0.0:1775767936.691004:0:8:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff9942103be000. 00000100:00080000:3.0:1775767936.691009:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000020:00000010:0.0:1775767936.691012:0:8:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0ec00. 00000020:00000010:0.0:1775767936.691022:0:8:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210b0f600. 00000100:00000040:3.0:1775767936.691026:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000010:0.0:1775767936.691033:0:8:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942078a3a20. 00000100:00000001:3.0:1775767936.691037:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000008:00000010:0.0:1775767936.691044:0:8:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff9942086e4e18. 00000100:00000001:1.0:1775767936.691044:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767936.691044:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767936.691055:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00020000:00000010:0.0:1775767936.691056:0:8:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994209592e40. 00000100:00000040:1.0:1775767936.691061:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a8a80 x1862022294253440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767961 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775767936.691061:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00008000:00000010:0.0:1775767936.691064:0:8:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994220358000. 00000020:00000040:3.0:1775767936.691065:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:3.0:1775767936.691072:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000080:00000010:0.0:1775767936.691075:0:8:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994207e3e3f0. 00000020:00000040:3.0:1775767936.691078:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000080:00000010:0.0:1775767936.691085:0:8:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b0ff8. 00000020:00000001:3.0:1775767936.691089:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:1.0:1775767936.691090:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000010:0.0:1775767936.691094:0:8:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0e000. 00000020:00000001:3.0:1775767936.691100:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:1.0:1775767936.691101:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00000010:0.0:1775767936.691102:0:8:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff99420906a6e0. 00000100:00000001:1.0:1775767936.691108:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767936.691115:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00020000:00000010:0.0:1775767936.691116:0:8:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff9942103cc480. 00000100:00000040:3.0:1775767936.691122:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00008000:00000010:0.0:1775767936.691124:0:8:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076adb0. 00000100:00000001:1.0:1775767936.691125:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767936.691126:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775767936.691131:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000080:00000010:0.0:1775767936.691132:0:8:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff994210769870. 00000100:00000040:3.0:1775767936.691139:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000001:0.0:1775767936.691140:0:8:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107752091656 : -112965957459960 : ffff9942103be008) 00000100:00080000:1.0:1775767936.691141:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000020:00001000:0.0:1775767936.691152:0:8:0:(cl_object.c:831:cl_env_get()) 1@ffff9942103be000 00000100:00080000:1.0:1775767936.691152:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767936.691153:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:1.0:1775767936.691176:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775767936.691181:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a8a80 x1862022294253440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767961 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767936.691187:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294253440:192.168.202.156@tcp:250:kworker.0 00000020:00000001:0.0:1775767936.691191:0:8:0:(cl_object.c:151:cl_object_top()) ffff9942078a75f8 -> ffff9942078a7f58 00000100:00000001:1.0:1775767936.691193:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1775767936.691199:0:8:0:(cl_io.c:125:__cl_io_init()) Process entered 00000100:00000001:1.0:1775767936.691202:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000020:00000040:3.0:1775767936.691202:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000020:00000001:0.0:1775767936.691203:0:8:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1775767936.691208:0:8:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00000040:1.0:1775767936.691211:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a8a80 x1862022294253440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767961 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775767936.691212:0:8:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767936.691218:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00020000:00000001:0.0:1775767936.691220:0:8:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767936.691229:0:8:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107752153128 : -112965957398488 : ffff9942103cd028) 00000100:00000001:3.0:1775767936.691232:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767936.691233:0:8:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000020:00000001:0.0:1775767936.691240:0:8:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767936.691243:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00020000:00000001:0.0:1775767936.691245:0:8:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767936.691250:0:8:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767936.691250:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775767936.691252:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767936.691260:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767936.691261:0:8:0:(cl_io.c:314:cl_io_lock()) Process entered 02000000:00000001:1.0:1775767936.691261:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00020000:00000001:0.0:1775767936.691266:0:8:0:(lov_io.c:1108:lov_io_lock()) Process entered 02000000:00000010:1.0:1775767936.691268:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543dc00. 00020000:00000001:0.0:1775767936.691273:0:8:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000001:3.0:1775767936.691274:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:1.0:1775767936.691275:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775767936.691280:0:8:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:1.0:1775767936.691284:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000001:0.0:1775767936.691285:0:8:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000008:00100000:3.0:1775767936.691285:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000020:00000040:1.0:1775767936.691288:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:0.0:1775767936.691291:0:8:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775767936.691297:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 00000008:00000001:3.0:1775767936.691301:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767936.691303:0:8:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775767936.691304:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b2c00. 00000100:00000001:3.0:1775767936.691307:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767936.691309:0:8:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767936.691310:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000010:3.0:1775767936.691313:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aa680. 00020000:00000001:0.0:1775767936.691314:0:8:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767936.691314:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775767936.691319:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a8a80. 00000020:00000001:0.0:1775767936.691323:0:8:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000040:3.0:1775767936.691324:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00000100:00000001:1.0:1775767936.691326:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000020:00000001:0.0:1775767936.691327:0:8:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767936.691331:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767936.691337:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1775767936.691337:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000020:00000001:0.0:1775767936.691338:0:8:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767936.691344:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767936.691345:0:8:0:(cl_io.c:528:cl_io_start()) Process entered 00020000:00000001:0.0:1775767936.691348:0:8:0:(lov_io.c:1114:lov_io_start()) Process entered 00000100:00000001:3.0:1775767936.691349:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00020000:00000001:0.0:1775767936.691351:0:8:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000001:1.0:1775767936.691351:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:3.0:1775767936.691353:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1775767936.691354:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767936.691356:0:8:0:(cl_io.c:528:cl_io_start()) Process entered 02000000:00000001:3.0:1775767936.691360:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1775767936.691361:0:8:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000100:00000001:1.0:1775767936.691362:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00000001:0.0:1775767936.691367:0:8:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000100:00000001:1.0:1775767936.691371:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000008:00000001:0.0:1775767936.691373:0:8:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000100:00000001:1.0:1775767936.691375:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 02000000:00000010:3.0:1775767936.691375:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2c00. 00000008:00000020:0.0:1775767936.691376:0:8:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:0.0:1775767936.691382:0:8:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1775767936.691386:0:8:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000008:00000001:0.0:1775767936.691388:0:8:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000100:00000001:3.0:1775767936.691388:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1775767936.691391:0:8:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107884688256 : -112965824863360 : ffff994218232380) 00000100:00000040:1.0:1775767936.691393:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099aa300 x1862022294253568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000020:0.0:1775767936.691400:0:8:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994218232380 ready 0|-|- wr 1|-|+ rd 0|- 9 in flight 00000100:00100000:3.0:1775767936.691406:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aa680 x1862022294253824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000008:00000001:0.0:1775767936.691407:0:8:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:0.0:1775767936.691412:0:8:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:0.0:1775767936.691417:0:8:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1775767936.691427:0:8:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000008:00000020:0.0:1775767936.691432:0:8:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994210674fc0 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000008:00000001:0.0:1775767936.691451:0:8:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767936.691453:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767936.691458:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1775767936.691463:0:8:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767936.691467:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:0.0:1775767936.691471:0:8:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767936.691472:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767936.691472:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00020000:00000001:0.0:1775767936.691480:0:8:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767936.691484:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767936.691485:0:8:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767936.691485:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767936.691488:0:8:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767936.691490:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006cb1597d] to pc [ptlrpcd_rcv+-1] req@ffff9942099aa680 x1862022294253824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775767936.691494:0:8:0:(cl_io.c:555:cl_io_end()) Process entered 00020000:00000001:0.0:1775767936.691500:0:8:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00000100:00100000:1.0:1775767936.691502:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099aa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294253568:192.168.202.156@tcp:38:kworker.0 00020000:00000001:0.0:1775767936.691516:0:8:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000020:00000001:0.0:1775767936.691520:0:8:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000001:0.0:1775767936.691525:0:8:0:(cl_io.c:563:cl_io_end()) Process leaving 00020000:00000001:0.0:1775767936.691531:0:8:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767936.691535:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00020000:00000001:0.0:1775767936.691543:0:8:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 02000000:00000001:1.0:1775767936.691546:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:0.0:1775767936.691547:0:8:0:(cl_io.c:563:cl_io_end()) Process leaving 00000100:00000001:3.0:1775767936.691549:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767936.691550:0:8:0:(cl_io.c:352:cl_io_unlock()) Process entered 02000000:00000001:1.0:1775767936.691550:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767936.691559:0:8:0:(lov_io.c:1208:lov_io_unlock()) Process entered 02000000:00000001:1.0:1775767936.691561:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00020000:00000001:0.0:1775767936.691562:0:8:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00080000:3.0:1775767936.691563:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000001:0.0:1775767936.691567:0:8:0:(cl_io.c:352:cl_io_unlock()) Process entered 02000000:00000010:1.0:1775767936.691567:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543dc00. 00000020:00000001:0.0:1775767936.691572:0:8:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:0.0:1775767936.691576:0:8:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767936.691579:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767936.691579:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00020000:00000001:0.0:1775767936.691580:0:8:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767936.691586:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00020000:00000001:0.0:1775767936.691588:0:8:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000100:00080000:3.0:1775767936.691590:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000020:00000001:0.0:1775767936.691591:0:8:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:0.0:1775767936.691597:0:8:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000400:00000200:1.0:1775767936.691601:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:0.0:1775767936.691603:0:8:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000100:00000001:3.0:1775767936.691605:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000020:00000001:0.0:1775767936.691606:0:8:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000400:00000010:1.0:1775767936.691607:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023cc08. 00020000:00000001:0.0:1775767936.691609:0:8:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00000100:00080000:3.0:1775767936.691613:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5189 00020000:00000001:0.0:1775767936.691614:0:8:0:(lov_io.c:1086:lov_io_call()) Process entered 00000400:00000010:1.0:1775767936.691619:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf67f8. 00000020:00000001:0.0:1775767936.691620:0:8:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000001:0.0:1775767936.691623:0:8:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00020000:00000001:0.0:1775767936.691625:0:8:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775767936.691628:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294253568, portal 10 00000400:00000200:3.0:1775767936.691632:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:0.0:1775767936.691634:0:8:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767936.691643:0:8:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000100:00000001:1.0:1775767936.691644:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000020:00000001:0.0:1775767936.691645:0:8:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000400:00000200:3.0:1775767936.691646:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000020:00000001:0.0:1775767936.691652:0:8:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767936.691652:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640861440 : -112966068690176 : ffff9942099aa300) 00000100:00080000:3.0:1775767936.691653:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000020:00000001:0.0:1775767936.691664:0:8:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1775767936.691668:0:8:0:(lov_io.c:675:lov_io_fini()) Process entered 00000100:00000040:3.0:1775767936.691669:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00020000:00000001:0.0:1775767936.691670:0:8:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:0.0:1775767936.691674:0:8:0:(cl_io.c:71:cl_io_fini()) Process entered 00000100:00000040:1.0:1775767936.691675:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099aa300 x1862022294253568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767961 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775767936.691678:0:8:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000100:00000001:3.0:1775767936.691679:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00001000:0.0:1775767936.691681:0:8:0:(cl_object.c:925:cl_env_put()) 1@ffff9942103be000 00000020:00000010:0.0:1775767936.691686:0:8:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210b0f600. 00000100:00000040:3.0:1775767936.691692:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000010:0.0:1775767936.691695:0:8:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942078a3a20. 00000008:00000010:0.0:1775767936.691702:0:8:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff9942086e4e18. 00000100:00000001:3.0:1775767936.691704:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:1.0:1775767936.691706:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00020000:00000010:0.0:1775767936.691708:0:8:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994209592e40. 00000020:00000001:3.0:1775767936.691717:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000040:1.0:1775767936.691718:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00008000:00000010:0.0:1775767936.691719:0:8:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994220358000. 00000400:00000010:1.0:1775767936.691726:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000080:00000010:0.0:1775767936.691728:0:8:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994207e3e3f0. 00000020:00000040:3.0:1775767936.691728:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000100:00000200:1.0:1775767936.691730:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294253568, offset 0 00000020:00000001:3.0:1775767936.691738:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000080:00000010:0.0:1775767936.691740:0:8:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b0ff8. 00000020:00000040:3.0:1775767936.691744:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000400:00000200:1.0:1775767936.691745:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000020:00000010:0.0:1775767936.691750:0:8:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0ec00. 00000020:00000001:3.0:1775767936.691755:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000008:00000010:0.0:1775767936.691757:0:8:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff99420906a6e0. 00000400:00000200:1.0:1775767936.691765:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:3.0:1775767936.691766:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00020000:00000010:0.0:1775767936.691768:0:8:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff9942103cc480. 00008000:00000010:0.0:1775767936.691781:0:8:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076adb0. 00000400:00000200:1.0:1775767936.691782:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00000040:3.0:1775767936.691785:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767936.691792:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000080:00000010:0.0:1775767936.691793:0:8:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff994210769870. 00000400:00000200:1.0:1775767936.691795:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000010:0.0:1775767936.691807:0:8:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0e000. 00000400:00000200:1.0:1775767936.691808:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000100:00000040:3.0:1775767936.691808:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767936.691819:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000200:1.0:1775767936.691820:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000020:00000010:0.0:1775767936.691821:0:8:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff9942103be000. 00000020:00000040:3.0:1775767936.691831:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00020000:00000001:0.0:1775767936.691834:0:8:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00000400:00000200:1.0:1775767936.691839:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00020000:00000001:0.0:1775767936.691847:0:8:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000100:00080000:3.0:1775767936.691849:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000080:00000001:0.0:1775767936.691856:0:8:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000100:00000001:3.0:1775767936.691856:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767936.691858:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225154:225154:256:4294967295] 192.168.202.156@tcp LPNI seq info [225150:225150:8:4294967295] 00000080:00200000:0.0:1775767936.691865:0:8:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 02000000:00000001:3.0:1775767936.691867:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767936.691873:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000001:0.0:1775767936.691880:0:8:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 02000000:00000001:3.0:1775767936.691881:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767936.691884:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:0.0:1775767936.691885:0:8:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000100:00000001:3.0:1775767936.691885:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000020:00001000:0.0:1775767936.691894:0:8:0:(cl_object.c:925:cl_env_put()) 1@ffff9942103bed80 00000008:00100000:3.0:1775767936.691894:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000020:00000010:0.0:1775767936.691902:0:8:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210b0fe00. 00000400:00000200:1.0:1775767936.691906:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000008:00000001:3.0:1775767936.691908:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1775767936.691910:0:8:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942078a3d80. 00000100:00000001:3.0:1775767936.691924:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:0.0:1775767936.691926:0:8:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff9942086e6468. 00000800:00000200:1.0:1775767936.691936:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00020000:00000010:0.0:1775767936.691937:0:8:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994209592f00. 00000100:00000010:3.0:1775767936.691937:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aad80. 00008000:00000010:0.0:1775767936.691948:0:8:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff99422035b880. 00000800:00000010:1.0:1775767936.691949:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541200. 00000020:00000040:3.0:1775767936.691949:0:13471:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00000080:00000010:0.0:1775767936.691954:0:8:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994207e3e150. 00000080:00000010:0.0:1775767936.691965:0:8:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b2238. 02000000:00000001:3.0:1775767936.691965:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000800:00000200:1.0:1775767936.691966:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000010:0.0:1775767936.691975:0:8:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0f200. 02000000:00000001:3.0:1775767936.691975:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775767936.691976:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000100:00000001:3.0:1775767936.691983:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000008:00000010:0.0:1775767936.691985:0:8:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff99420906be40. 02000000:00000001:3.0:1775767936.691987:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00020000:00000010:0.0:1775767936.691992:0:8:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff9942103ccfc0. 02000000:00000001:3.0:1775767936.691994:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775767936.691994:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00008000:00000010:0.0:1775767936.691997:0:8:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076afa0. 00000800:00000200:1.1:1775767936.692011:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000080:00000010:0.0:1775767936.692013:0:8:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942107690f0. 02000000:00000010:3.0:1775767936.692014:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0400. 00000800:00000100:1.0:1775767936.692025:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000020:00000010:0.0:1775767936.692027:0:8:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0f800. 00000100:00000001:3.0:1775767936.692027:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1775767936.692028:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541200. 00000020:00000010:0.0:1775767936.692043:0:8:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff9942103bed80. 00000400:00000200:1.0:1775767936.692043:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00100000:3.0:1775767936.692047:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aad80 x1862022294253952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000080:00000001:0.0:1775767936.692053:0:8:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1775767936.692056:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000080:00000001:0.0:1775767936.692066:0:8:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767936.692070:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000400:00000010:1.0:1775767936.692077:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000080:00000001:0.0:1775767936.692082:0:8:0:(rw.c:1481:ll_writepages()) Process entered 00000100:00000001:3.0:1775767936.692083:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775767936.692084:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000080:00000001:0.0:1775767936.692089:0:8:0:(file.c:5250:cl_sync_file_range()) Process entered 00000100:00000001:3.0:1775767936.692089:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767936.692094:0:8:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00000200:1.0:1775767936.692098:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099aa300 x1862022294253568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767961 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1775767936.692100:0:8:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff9942103bed80. 00000100:00000040:3.0:1775767936.692100:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000045ccf08] to pc [ptlrpcd_rcv+-1] req@ffff9942099aad80 x1862022294253952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1775767936.692111:0:8:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0f800. 00000020:00000010:0.0:1775767936.692122:0:8:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210b0f200. 00000020:00000010:0.0:1775767936.692131:0:8:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942078a3d80. 00000008:00000010:0.0:1775767936.692139:0:8:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff9942086e6468. 00020000:00000010:0.0:1775767936.692146:0:8:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994209592f00. 00008000:00000010:0.0:1775767936.692175:0:8:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff99422035b880. 00000080:00000010:0.0:1775767936.692179:0:8:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994207e3e150. 00000080:00000010:0.0:1775767936.692182:0:8:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b2238. 00000020:00000010:0.0:1775767936.692184:0:8:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0fe00. 00000008:00000010:0.0:1775767936.692191:0:8:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff99420906be40. 00000100:00000001:1.0:1775767936.692197:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00020000:00000010:0.0:1775767936.692199:0:8:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff9942103ccfc0. 00008000:00000010:0.0:1775767936.692203:0:8:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076afa0. 00000100:00000001:3.0:1775767936.692204:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767936.692208:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099aa300 x1862022294253568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767961 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000010:0.0:1775767936.692209:0:8:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942107690f0. 00000020:00000001:0.0:1775767936.692215:0:8:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107752095112 : -112965957456504 : ffff9942103bed88) 00000100:00000040:3.0:1775767936.692217:0:13471:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5199) 00000020:00001000:0.0:1775767936.692225:0:8:0:(cl_object.c:831:cl_env_get()) 1@ffff9942103bed80 00000020:00000001:0.0:1775767936.692229:0:8:0:(cl_object.c:151:cl_object_top()) ffff99420791a850 -> ffff99420791a850 00000100:00000001:1.0:1775767936.692230:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767936.692234:0:8:0:(cl_io.c:125:__cl_io_init()) Process entered 00000100:00000001:1.0:1775767936.692234:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000080:00000001:0.0:1775767936.692238:0:8:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:0.0:1775767936.692241:0:8:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017f:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000100:00000001:1.0:1775767936.692242:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767936.692244:0:8:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1775767936.692246:0:8:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1775767936.692247:0:8:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767936.692248:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00020000:00000002:0.0:1775767936.692250:0:8:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017f:0x0]io ffff9942077b2298 type 6 ignore/verify layout 1/0 00020000:00000002:0.0:1775767936.692254:0:8:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207dfbd90) owner 0000000000000000/ffff994201224f80 00000400:00000200:1.0:1775767936.692255:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:0.0:1775767936.692260:0:8:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00000100:00000001:1.0:1775767936.692260:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299861525 00020000:00000001:0.0:1775767936.692262:0:8:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1775767936.692266:0:8:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00000100:00000001:1.0:1775767936.692266:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299862846 00020000:00000001:0.0:1775767936.692269:0:8:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767936.692270:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299861525 00020000:00000001:0.0:1775767936.692272:0:8:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775767936.692273:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299862846 00020000:00000001:0.0:1775767936.692277:0:8:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00000100:00000001:1.0:1775767936.692277:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1775767936.692280:0:8:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767936.692281:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767936.692283:0:8:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000100:00000001:1.0:1775767936.692284:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000020:00000001:0.0:1775767936.692285:0:8:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1775767936.692287:0:8:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00020000:00000002:0.0:1775767936.692289:0:8:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207dfbd90) owner 0000000000000000/ffff994201224f80 00000100:00100000:1.0:1775767936.692289:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767936/real 1775767936] req@ffff9942099aa300 x1862022294253568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775767936.692294:0:8:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767936.692298:0:8:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:0.0:1775767936.692300:0:8:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00020000:00000001:0.0:1775767936.692303:0:8:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00020000:00200000:0.0:1775767936.692305:0:8:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000400:00000001:1.0:1775767936.692305:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775767936.692307:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00020000:00000002:0.0:1775767936.692308:0:8:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00000400:00000010:1.0:1775767936.692310:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023cc08. 00020000:00000001:0.0:1775767936.692311:0:8:0:(lov_io.c:155:lov_sub_get()) Process entered 00020000:00000001:0.0:1775767936.692314:0:8:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000400:00000200:1.0:1775767936.692314:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf67f8 00000020:00000001:0.0:1775767936.692316:0:8:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000400:00000010:1.0:1775767936.692317:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf67f8. 00000020:00000010:0.0:1775767936.692319:0:8:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff9942103be000. 00000100:00000001:1.0:1775767936.692320:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000010:0.0:1775767936.692323:0:8:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0e000. 00000100:00000200:1.0:1775767936.692326:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099aa300 x1862022294253568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1775767936.692327:0:8:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210b0ec00. 00000020:00000010:0.0:1775767936.692331:0:8:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942078a3a20. 00000008:00000010:0.0:1775767936.692335:0:8:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff9942086e4e18. 00020000:00000010:0.0:1775767936.692339:0:8:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994209592e40. 00000100:00000200:1.0:1775767936.692341:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099aa300 x1862022294253568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00008000:00000010:0.0:1775767936.692342:0:8:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994220358000. 00000080:00000010:0.0:1775767936.692347:0:8:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994207e3e3f0. 00000080:00000010:0.0:1775767936.692352:0:8:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b0ff8. 00000020:00000010:0.0:1775767936.692357:0:8:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0f600. 00000100:00000001:1.0:1775767936.692357:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000008:00000010:0.0:1775767936.692361:0:8:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff99420906a6e0. 00000100:00000001:1.0:1775767936.692361:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00020000:00000010:0.0:1775767936.692366:0:8:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff9942103cc480. 00008000:00000010:0.0:1775767936.692368:0:8:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076adb0. 00000080:00000010:0.0:1775767936.692369:0:8:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff994210769870. 00000020:00000001:0.0:1775767936.692370:0:8:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107752091656 : -112965957459960 : ffff9942103be008) 00000020:00001000:0.0:1775767936.692372:0:8:0:(cl_object.c:831:cl_env_get()) 1@ffff9942103be000 00000020:00000001:0.0:1775767936.692373:0:8:0:(cl_object.c:151:cl_object_top()) ffff99420791a210 -> ffff99420791a850 00000020:00000001:0.0:1775767936.692375:0:8:0:(cl_io.c:125:__cl_io_init()) Process entered 00000100:00000001:1.0:1775767936.692375:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:0.0:1775767936.692379:0:8:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1775767936.692380:0:8:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:0.0:1775767936.692380:0:8:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767936.692382:0:8:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767936.692383:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00020000:00000001:0.0:1775767936.692385:0:8:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107752153128 : -112965957398488 : ffff9942103cd028) 00000020:00000001:0.0:1775767936.692388:0:8:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000020:00000001:0.0:1775767936.692390:0:8:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767936.692391:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099aa300 x1862022294253568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:0.0:1775767936.692392:0:8:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767936.692396:0:8:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767936.692400:0:8:0:(cl_io.c:314:cl_io_lock()) Process entered 00020000:00000001:0.0:1775767936.692401:0:8:0:(lov_io.c:1108:lov_io_lock()) Process entered 00020000:00000001:0.0:1775767936.692403:0:8:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:0.0:1775767936.692406:0:8:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:0.0:1775767936.692407:0:8:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:0.0:1775767936.692409:0:8:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767936.692412:0:8:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767936.692412:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00020000:00000001:0.0:1775767936.692417:0:8:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767936.692419:0:8:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767936.692420:0:8:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:0.0:1775767936.692422:0:8:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767936.692423:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099aa300 x1862022294253568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767961 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775767936.692424:0:8:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767936.692427:0:8:0:(cl_io.c:528:cl_io_start()) Process entered 00020000:00000001:0.0:1775767936.692429:0:8:0:(lov_io.c:1114:lov_io_start()) Process entered 00020000:00000001:0.0:1775767936.692431:0:8:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:0.0:1775767936.692432:0:8:0:(cl_io.c:528:cl_io_start()) Process entered 00000008:00000001:0.0:1775767936.692435:0:8:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000008:00000001:0.0:1775767936.692437:0:8:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000008:00000001:0.0:1775767936.692440:0:8:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000100:00000001:1.0:1775767936.692441:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00000020:0.0:1775767936.692442:0:8:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:0.0:1775767936.692445:0:8:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767936.692446:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1775767936.692448:0:8:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000100:00000001:1.0:1775767936.692450:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000008:00000001:0.0:1775767936.692451:0:8:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000100:00000001:1.0:1775767936.692454:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1775767936.692455:0:8:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107754938304 : -112965954613312 : ffff994210674fc0) 00000100:00000001:1.0:1775767936.692456:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000008:00000020:0.0:1775767936.692460:0:8:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994210674fc0 ready 0|-|- wr 1|-|+ rd 0|- 9 in flight 00000100:00000001:1.0:1775767936.692460:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00000001:0.0:1775767936.692468:0:8:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000100:00080000:1.0:1775767936.692468:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00000020:0.0:1775767936.692471:0:8:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:0.0:1775767936.692474:0:8:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1775767936.692476:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000008:00000001:0.0:1775767936.692479:0:8:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000100:00000001:1.0:1775767936.692481:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000008:00000020:0.0:1775767936.692482:0:8:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994218232380 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000008:00000001:0.0:1775767936.692490:0:8:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775767936.692490:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099aa300 x1862022294253568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767961 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:0.0:1775767936.692495:0:8:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767936.692500:0:8:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767936.692504:0:8:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767936.692521:0:8:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767936.692524:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099aa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294253568:192.168.202.156@tcp:38:kworker.0 00000020:00000001:0.0:1775767936.692526:0:8:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767936.692530:0:8:0:(cl_io.c:555:cl_io_end()) Process entered 00020000:00000001:0.0:1775767936.692533:0:8:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00000100:00000001:1.0:1775767936.692535:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00020000:00000001:0.0:1775767936.692537:0:8:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000020:00000001:0.0:1775767936.692540:0:8:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000001:0.0:1775767936.692541:0:8:0:(cl_io.c:563:cl_io_end()) Process leaving 00000100:00000040:1.0:1775767936.692543:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ab800 x1862022294253696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:0.0:1775767936.692544:0:8:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767936.692549:0:8:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000020:00000001:0.0:1775767936.692552:0:8:0:(cl_io.c:563:cl_io_end()) Process leaving 00000020:00000001:0.0:1775767936.692554:0:8:0:(cl_io.c:352:cl_io_unlock()) Process entered 00020000:00000001:0.0:1775767936.692556:0:8:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00020000:00000001:0.0:1775767936.692559:0:8:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:0.0:1775767936.692561:0:8:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000100:00000001:1.0:1775767936.692562:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000001:0.0:1775767936.692563:0:8:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:0.0:1775767936.692567:0:8:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767936.692567:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767936.692573:0:8:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767936.692574:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00020000:00000001:0.0:1775767936.692577:0:8:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 02000000:00000001:1.0:1775767936.692578:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1775767936.692580:0:8:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:0.0:1775767936.692583:0:8:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 02000000:00000001:1.0:1775767936.692584:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767936.692586:0:8:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000001:0.0:1775767936.692589:0:8:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:0.0:1775767936.692590:0:8:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00020000:00000001:0.0:1775767936.692592:0:8:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00100000:1.0:1775767936.692592:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294253696:192.168.202.156@tcp:38:kworker.0 00000020:00000001:0.0:1775767936.692594:0:8:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000001:0.0:1775767936.692596:0:8:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00020000:00000001:0.0:1775767936.692598:0:8:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767936.692602:0:8:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767936.692602:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00020000:00000001:0.0:1775767936.692603:0:8:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000020:00000001:0.0:1775767936.692605:0:8:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 02000000:00000001:1.0:1775767936.692606:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:0.0:1775767936.692607:0:8:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767936.692610:0:8:0:(cl_io.c:71:cl_io_fini()) Process entered 02000000:00000001:1.0:1775767936.692610:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767936.692613:0:8:0:(lov_io.c:675:lov_io_fini()) Process entered 02000000:00000001:1.0:1775767936.692613:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00020000:00000001:0.0:1775767936.692614:0:8:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:0.0:1775767936.692616:0:8:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:0.0:1775767936.692617:0:8:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1775767936.692618:0:8:0:(cl_object.c:925:cl_env_put()) 1@ffff9942103be000 02000000:00000010:1.0:1775767936.692619:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543d800. 00000020:00000010:0.0:1775767936.692620:0:8:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210b0ec00. 02000000:00000001:1.0:1775767936.692625:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1775767936.692626:0:8:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942078a3a20. 00000008:00000010:0.0:1775767936.692632:0:8:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff9942086e4e18. 00000400:00000200:1.0:1775767936.692633:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000010:0.0:1775767936.692637:0:8:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994209592e40. 00000400:00000010:1.0:1775767936.692638:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023cc08. 00008000:00000010:0.0:1775767936.692642:0:8:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994220358000. 00000400:00000010:1.0:1775767936.692644:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf67f8. 00000080:00000010:0.0:1775767936.692646:0:8:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994207e3e3f0. 00000100:00000200:1.0:1775767936.692650:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294253696, portal 10 00000080:00000010:0.0:1775767936.692651:0:8:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b0ff8. 00000100:00000001:1.0:1775767936.692657:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000020:00000010:0.0:1775767936.692658:0:8:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0e000. 00000100:00000001:1.0:1775767936.692660:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640866816 : -112966068684800 : ffff9942099ab800) 00000008:00000010:0.0:1775767936.692663:0:8:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff99420906a6e0. 00020000:00000010:0.0:1775767936.692668:0:8:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff9942103cc480. 00000100:00000040:1.0:1775767936.692670:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ab800 x1862022294253696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767961 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00008000:00000010:0.0:1775767936.692673:0:8:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076adb0. 00000080:00000010:0.0:1775767936.692678:0:8:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff994210769870. 00000020:00000010:0.0:1775767936.692683:0:8:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0f600. 00000100:00000001:1.0:1775767936.692685:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000010:0.0:1775767936.692688:0:8:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff9942103be000. 00000100:00000040:1.0:1775767936.692692:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00020000:00000001:0.0:1775767936.692693:0:8:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:0.0:1775767936.692696:0:8:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000400:00000010:1.0:1775767936.692697:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000080:00000001:0.0:1775767936.692698:0:8:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000080:00200000:0.0:1775767936.692701:0:8:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017f:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00000200:1.0:1775767936.692703:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294253696, offset 0 00000080:00000001:0.0:1775767936.692704:0:8:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1775767936.692707:0:8:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1775767936.692710:0:8:0:(cl_object.c:925:cl_env_put()) 1@ffff9942103bed80 00000020:00000010:0.0:1775767936.692713:0:8:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210b0f200. 00000400:00000200:1.0:1775767936.692714:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000020:00000010:0.0:1775767936.692717:0:8:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942078a3d80. 00000008:00000010:0.0:1775767936.692721:0:8:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff9942086e6468. 00000400:00000200:1.0:1775767936.692724:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000010:0.0:1775767936.692725:0:8:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994209592f00. 00008000:00000010:0.0:1775767936.692729:0:8:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff99422035b880. 00000400:00000200:1.0:1775767936.692732:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000080:00000010:0.0:1775767936.692733:0:8:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994207e3e150. 00000080:00000010:0.0:1775767936.692738:0:8:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b2238. 00000400:00000200:1.0:1775767936.692738:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000010:0.0:1775767936.692742:0:8:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0f800. 00000400:00000200:1.0:1775767936.692743:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000008:00000010:0.0:1775767936.692746:0:8:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff99420906be40. 00000400:00000200:1.0:1775767936.692748:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00020000:00000010:0.0:1775767936.692750:0:8:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff9942103ccfc0. 00000400:00000200:1.0:1775767936.692752:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00008000:00000010:0.0:1775767936.692754:0:8:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076afa0. 00000400:00000200:1.0:1775767936.692758:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225155:225155:256:4294967295] 192.168.202.156@tcp LPNI seq info [225151:225151:8:4294967295] 00000080:00000010:0.0:1775767936.692759:0:8:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942107690f0. 00000020:00000010:0.0:1775767936.692765:0:8:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0fe00. 00000400:00000200:1.0:1775767936.692766:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000010:0.0:1775767936.692771:0:8:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff9942103bed80. 00000080:00000001:0.0:1775767936.692777:0:8:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1775767936.692778:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000080:00000001:0.0:1775767936.692783:0:8:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775767936.692786:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767936.692790:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541200. 00000800:00000200:1.0:1775767936.692794:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775767936.692797:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775767936.692801:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767936.692803:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1775767936.692808:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767936.692809:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541200. 00000400:00000200:1.0:1775767936.692812:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775767936.692816:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775767936.692819:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000400:00000010:1.0:1775767936.692821:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000100:00000001:1.0:1775767936.692825:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775767936.692829:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ab800 x1862022294253696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767961 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767936.692843:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767936.692846:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ab800 x1862022294253696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767961 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767936.692854:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767936.692855:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775767936.692860:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767936.692865:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:1.0:1775767936.692869:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767936.692871:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299861517 00000100:00000001:1.0:1775767936.692874:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299862846 00000100:00000001:1.0:1775767936.692877:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299861517 00000100:00000001:1.0:1775767936.692880:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299862846 00000100:00000001:1.0:1775767936.692883:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775767936.692886:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767936.692888:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775767936.692893:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767936/real 1775767936] req@ffff9942099ab800 x1862022294253696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775767936.692902:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775767936.692904:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775767936.692906:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023cc08. 00000400:00000200:1.0:1775767936.692909:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf67f8 00000400:00000010:1.0:1775767936.692911:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf67f8. 00000100:00000001:1.0:1775767936.692915:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775767936.692919:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ab800 x1862022294253696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775767936.692929:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ab800 x1862022294253696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767936.692934:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775767936.692936:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767936.692938:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767936.692939:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767936.692942:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ab800 x1862022294253696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767936.692946:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775767936.692948:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ab800 x1862022294253696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767961 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767936.692952:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775767936.692954:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767936.692955:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767936.692956:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767936.692958:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775767936.692961:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775767936.692963:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767936.692965:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775767936.692966:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775767936.692969:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ab800 x1862022294253696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767961 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767936.692974:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294253696:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775767936.692977:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767936.692979:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767936.692981:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099aa300 x1862022294253568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767961 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767936.692985:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775767936.692986:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775767936.692988:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543dc00. 02000000:00000001:1.0:1775767936.692990:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775767936.692991:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775767936.692993:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:1.0:1775767936.692995:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775767936.692996:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b3400. 02000000:00000001:1.0:1775767936.693000:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775767936.693001:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775767936.693003:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099aa300. 00000100:00000001:1.0:1775767936.693005:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775767936.693006:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767936.693007:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767936.693009:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ab800 x1862022294253696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767961 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767936.693014:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775767936.693014:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775767936.693015:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543d800. 02000000:00000001:1.0:1775767936.693016:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775767936.693017:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775767936.693018:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:1.0:1775767936.693019:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775767936.693020:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b1000. 02000000:00000001:1.0:1775767936.693021:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775767936.693022:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775767936.693022:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ab800. 00000100:00000001:1.0:1775767936.693024:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775767936.693025:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767936.693026:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767936.693030:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767936.693032:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767936.693034:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775767936.693035:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767936.693036:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775767936.693038:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099aa680 x1862022294253824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767936.693043:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767936.693044:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767936.693045:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775767936.693047:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767936.693048:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767936.693050:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099aa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294253824:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775767936.693053:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775767936.693055:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775767936.693056:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767936.693057:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775767936.693060:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543d800. 02000000:00000001:1.0:1775767936.693061:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767936.693064:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775767936.693066:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023cc08. 00000400:00000010:1.0:1775767936.693068:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf67f8. 00000100:00000200:1.0:1775767936.693071:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294253824, portal 4 00000100:00000001:1.0:1775767936.693072:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775767936.693073:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640862336 : -112966068689280 : ffff9942099aa680) 00000100:00000040:1.0:1775767936.693077:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099aa680 x1862022294253824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767961 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767936.693082:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775767936.693083:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775767936.693085:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000100:00000200:1.0:1775767936.693086:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294253824, offset 0 00000400:00000200:1.0:1775767936.693089:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775767936.693091:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767936.693094:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775767936.693095:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775767936.693097:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775767936.693099:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775767936.693100:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775767936.693103:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225156:225156:256:4294967295] 192.168.202.156@tcp LPNI seq info [225152:225152:8:4294967295] 00000400:00000200:1.0:1775767936.693106:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767936.693111:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775767936.693114:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767936.693117:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541200. 00000800:00000200:1.0:1775767936.693120:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775767936.693122:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775767936.693125:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767936.693140:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1775767936.693143:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767936.693145:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541200. 00000400:00000200:1.0:1775767936.693147:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775767936.693149:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775767936.693152:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000400:00000010:1.0:1775767936.693153:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000100:00000001:1.0:1775767936.693174:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775767936.693178:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099aa680 x1862022294253824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767961 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767936.693186:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767936.693190:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099aa680 x1862022294253824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767961 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767936.693195:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767936.693197:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775767936.693202:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767936.693205:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:1.0:1775767936.693208:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767936.693209:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299861541 00000100:00000001:1.0:1775767936.693211:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299862846 00000100:00000001:1.0:1775767936.693213:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299861541 00000100:00000001:1.0:1775767936.693215:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299862846 00000100:00000001:1.0:1775767936.693216:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775767936.693217:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767936.693219:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775767936.693221:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767936/real 1775767936] req@ffff9942099aa680 x1862022294253824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775767936.693231:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775767936.693232:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775767936.693233:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023cc08. 00000400:00000200:1.0:1775767936.693235:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf67f8 00000400:00000010:1.0:1775767936.693236:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf67f8. 00000100:00000001:1.0:1775767936.693237:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775767936.693239:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099aa680 x1862022294253824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775767936.693245:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099aa680 x1862022294253824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767936.693249:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775767936.693250:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767936.693251:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767936.693252:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767936.693254:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099aa680 x1862022294253824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767936.693258:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775767936.693260:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099aa680 x1862022294253824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767961 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767936.693264:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775767936.693266:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767936.693266:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767936.693267:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767936.693268:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775767936.693271:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775767936.693273:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767936.693275:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775767936.693276:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775767936.693279:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099aa680 x1862022294253824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767961 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767936.693285:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099aa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294253824:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775767936.693288:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775767936.693290:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099aad80 x1862022294253952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767936.693294:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767936.693295:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767936.693296:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775767936.693297:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767936.693299:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767936.693300:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099aad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294253952:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775767936.693303:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775767936.693304:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775767936.693306:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767936.693307:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775767936.693309:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543dc00. 02000000:00000001:1.0:1775767936.693311:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767936.693312:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775767936.693314:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023cc08. 00000400:00000010:1.0:1775767936.693316:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf67f8. 00000100:00000200:1.0:1775767936.693318:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294253952, portal 4 00000100:00000001:1.0:1775767936.693319:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775767936.693320:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640864128 : -112966068687488 : ffff9942099aad80) 00000100:00000040:1.0:1775767936.693325:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099aad80 x1862022294253952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767961 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767936.693329:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775767936.693331:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775767936.693332:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000100:00000200:1.0:1775767936.693333:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294253952, offset 0 00000400:00000200:1.0:1775767936.693336:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775767936.693339:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767936.693341:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775767936.693343:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775767936.693345:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775767936.693347:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775767936.693349:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775767936.693351:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225157:225157:256:4294967295] 192.168.202.156@tcp LPNI seq info [225153:225153:8:4294967295] 00000400:00000200:1.0:1775767936.693354:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767936.693358:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775767936.693362:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767936.693365:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541200. 00000800:00000200:1.0:1775767936.693369:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775767936.693372:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775767936.693376:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775767936.693378:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1775767936.693382:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775767936.693384:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541200. 00000400:00000200:1.0:1775767936.693386:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775767936.693389:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775767936.693393:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000400:00000010:1.0:1775767936.693394:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000100:00000001:1.0:1775767936.693397:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775767936.693400:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099aad80 x1862022294253952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767961 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767936.693409:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767936.693412:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099aad80 x1862022294253952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767961 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767936.693419:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767936.693421:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775767936.693424:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767936.693427:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:1.0:1775767936.693430:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767936.693433:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299861544 00000100:00000001:1.0:1775767936.693435:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299862846 00000100:00000001:1.0:1775767936.693437:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299861544 00000100:00000001:1.0:1775767936.693438:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299862846 00000100:00000001:1.0:1775767936.693440:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775767936.693441:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767936.693442:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775767936.693445:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767936/real 1775767936] req@ffff9942099aad80 x1862022294253952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775767936.693450:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775767936.693451:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775767936.693453:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023cc08. 00000400:00000200:1.0:1775767936.693454:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf67f8 00000400:00000010:1.0:1775767936.693456:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf67f8. 00000100:00000001:1.0:1775767936.693457:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775767936.693459:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099aad80 x1862022294253952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775767936.693465:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099aad80 x1862022294253952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767936.693469:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775767936.693470:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767936.693471:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767936.693472:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767936.693475:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099aad80 x1862022294253952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767936.693482:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775767936.693485:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099aad80 x1862022294253952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767961 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767936.693492:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775767936.693495:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767936.693497:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767936.693498:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767936.693500:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775767936.693504:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775767936.693521:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767936.693525:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775767936.693528:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775767936.693532:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099aad80 x1862022294253952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767961 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767936.693541:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099aad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294253952:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775767936.693545:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767936.693548:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767936.693551:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099aa680 x1862022294253824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767961 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767936.693557:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775767936.693559:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775767936.693560:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543d800. 02000000:00000001:1.0:1775767936.693562:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775767936.693563:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775767936.693564:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:1.0:1775767936.693565:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775767936.693567:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b2c00. 02000000:00000001:1.0:1775767936.693569:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775767936.693570:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775767936.693571:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099aa680. 00000100:00000001:1.0:1775767936.693573:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775767936.693573:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767936.693575:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767936.693577:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099aad80 x1862022294253952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767961 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767936.693586:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775767936.693587:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775767936.693588:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543dc00. 02000000:00000001:1.0:1775767936.693589:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775767936.693590:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775767936.693590:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:1.0:1775767936.693591:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775767936.693592:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b0400. 02000000:00000001:1.0:1775767936.693594:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775767936.693595:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775767936.693596:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099aad80. 00000100:00000001:1.0:1775767936.693597:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775767936.693598:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767936.693599:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767936.693602:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767936.693602:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767936.693604:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775767936.693605:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767936.753293:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767936.753300:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767936.753304:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767936.753305:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767936.753309:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767936.753311:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767936.753315:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767936.753316:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767936.753320:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767936.753321:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767936.753345:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767936.753348:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767936.753350:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767936.817306:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767936.817315:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767936.817317:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767936.817324:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767936.817329:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767936.817333:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767936.817336:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767936.817339:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767936.817343:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767937.713231:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767937.713240:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767937.713242:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767937.713243:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767937.713243:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767937.713249:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767937.713253:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767937.713254:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:3.0:1775767937.713254:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767937.713260:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767937.713267:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:1.0:1775767937.713268:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767937.713268:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767937.713279:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767937.713280:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767937.713281:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767937.713284:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767937.713286:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767937.713288:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767937.713294:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767937.713297:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767937.713300:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767937.713303:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767937.713306:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767937.713307:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767937.713310:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767937.713313:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767937.777280:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767937.777288:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767937.777297:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767937.777298:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767937.777303:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767937.777304:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767937.777309:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767937.777310:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767937.777318:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767937.777319:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767937.777342:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767937.777345:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767937.777347:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767937.841280:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767937.841288:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767937.841290:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767937.841296:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767937.841300:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767937.841304:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767937.841307:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767937.841310:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767937.841312:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767938.033272:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299862846, at 4299862848 00000100:00000040:0.0:1775767938.033277:0:14953:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4299862846, at 4299862848 00000100:00000010:0.0:1775767938.033292:0:14953:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cba600. 00000100:00000010:1.0:1775767938.033302:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cba9c0. 00000100:00000001:1.0:1775767938.737287:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767938.737287:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767938.737295:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767938.737296:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767938.737297:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767938.737300:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767938.737304:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775767938.737304:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767938.737309:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767938.737310:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767938.737315:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767938.737316:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767938.737321:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767938.737322:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767938.737329:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767938.737329:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767938.737333:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767938.737341:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767938.737342:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00100000:2.0:1775767938.737342:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767938.737350:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767938.737351:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767938.737358:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767938.737359:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767938.737368:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767938.737371:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767938.737374:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767938.801270:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767938.801277:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767938.801284:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767938.801286:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767938.801290:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767938.801292:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767938.801296:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767938.801297:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767938.801301:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767938.801303:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767938.801324:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767938.801327:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767938.801330:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767938.865241:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767938.865247:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767938.865249:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767938.865253:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767938.865256:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767938.865259:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767938.865261:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767938.865263:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767938.865270:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767939.761243:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767939.761249:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767939.761251:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767939.761257:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767939.761259:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767939.761261:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767939.761263:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767939.761265:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767939.761266:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767939.761269:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767939.761269:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767939.761272:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767939.761273:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767939.761277:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767939.761277:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767939.761286:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767939.761288:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767939.761292:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767939.761315:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767939.761323:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767939.761325:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767939.761329:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775767939.761333:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767939.761335:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767939.761338:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767939.761340:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767939.761343:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767939.825240:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767939.825245:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767939.825252:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767939.825254:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767939.825258:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767939.825259:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767939.825262:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767939.825263:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767939.825266:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767939.825267:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767939.825286:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767939.825289:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767939.825291:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767939.889259:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767939.889265:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767939.889267:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767939.889272:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767939.889275:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767939.889278:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767939.889280:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767939.889282:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767939.889284:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767940.785249:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767940.785249:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767940.785249:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767940.785255:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767940.785256:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767940.785256:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767940.785260:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767940.785260:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767940.785261:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767940.785268:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:0.0:1775767940.785269:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:1.0:1775767940.785269:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:0.0:1775767940.785276:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767940.785276:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767940.785276:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767940.785281:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767940.785281:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767940.785283:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767940.785286:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767940.785289:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767940.785291:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767940.785294:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767940.785297:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767940.785301:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767940.785301:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767940.785304:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767940.785307:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767940.849245:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767940.849251:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775767940.849253:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775767940.849256:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767940.849260:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775767940.849262:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767940.849263:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767940.849265:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767940.849267:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767940.849269:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767940.849271:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767940.849272:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767940.849275:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767940.849275:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767940.849278:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767940.849279:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767940.849292:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767940.849294:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767940.849296:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767940.913225:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767940.913231:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767940.913232:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767940.913238:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767940.913241:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767940.913244:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767940.913245:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767940.913247:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767940.913249:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767941.809258:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767941.809266:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767941.809269:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767941.809271:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767941.809277:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:3.0:1775767941.809277:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775767941.809279:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767941.809285:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:0.0:1775767941.809286:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775767941.809291:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767941.809292:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767941.809296:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767941.809297:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767941.809301:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767941.809306:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767941.809308:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775767941.809310:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:1.0:1775767941.809315:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775767941.809316:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767941.809319:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1775767941.809319:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767941.809321:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767941.809323:0:8:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775767941.809326:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767941.809331:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767941.809332:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00100000:2.0:1775767941.809340:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775767941.809344:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:1.0:1775767941.809345:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5194 00000100:00000001:2.0:1775767941.809348:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767941.809352:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767941.809356:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767941.809359:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767941.809360:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767941.809372:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775767941.809375:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775767941.809379:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767941.809383:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767941.809386:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767941.809388:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775767941.809392:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775767941.809395:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:1.0:1775767941.809397:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775767941.809400:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:1.0:1775767941.809402:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:1.0:1775767941.809405:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:1.0:1775767941.809418:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767941.809421:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767941.809424:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767941.809426:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775767941.809429:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:1.0:1775767941.809434:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775767941.809437:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767941.809441:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775767941.809444:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767941.809447:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767941.809450:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775767941.809453:0:8:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775767941.809473:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207889880. 00000020:00000040:1.0:1775767941.809478:0:8:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:1.0:1775767941.809482:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775767941.809483:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767941.809486:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775767941.809489:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775767941.809492:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775767941.809499:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543d800. 00000100:00000001:1.0:1775767941.809505:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767941.809513:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207889880 x1862022294254080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775767941.809522:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775767941.809526:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767941.809530:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f11856b8] to pc [ptlrpcd_rcv+-1] req@ffff994207889880 x1862022294254080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767941.809553:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767941.809556:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767941.809559:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775767941.809560:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775767941.809561:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775767941.809563:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775767941.809567:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5194 00000400:00000200:1.0:1775767941.809572:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767941.809575:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775767941.809578:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775767941.809580:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767941.809583:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767941.809585:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767941.809587:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775767941.809589:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775767941.809591:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:1.0:1775767941.809592:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775767941.809594:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000100:00000001:0.0:1775767941.809596:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1775767941.809596:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:1.0:1775767941.809598:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:1.0:1775767941.809602:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775767941.809603:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767941.809604:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1775767941.809607:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775767941.809608:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767941.809611:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775767941.809613:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00080000:1.0:1775767941.809617:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000040:0.0:1775767941.809620:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207889880 x1862022294254080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767941.809620:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767941.809624:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775767941.809629:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767941.809633:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767941.809637:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767941.809637:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775767941.809641:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:1.0:1775767941.809643:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1775767941.809646:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000001:1.0:1775767941.809647:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767941.809649:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775767941.809650:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767941.809657:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775767941.809659:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207888e00. 00000020:00000040:1.0:1775767941.809663:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000100:00100000:0.0:1775767941.809664:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207889880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294254080:192.168.202.156@tcp:250:kworker.0 02000000:00000001:1.0:1775767941.809669:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1775767941.809673:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775767941.809673:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767941.809676:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775767941.809679:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775767941.809680:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775767941.809682:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767941.809684:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775767941.809688:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543e400. 02000000:00000001:0.0:1775767941.809689:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1775767941.809694:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775767941.809696:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220029400. 00000100:00100000:1.0:1775767941.809700:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207888e00 x1862022294254208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1775767941.809701:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767941.809710:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:0.0:1775767941.809712:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767941.809713:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767941.809718:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001812f966] to pc [ptlrpcd_rcv+-1] req@ffff994207888e00 x1862022294254208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1775767941.809719:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775767941.809728:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000001:1.0:1775767941.809728:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767941.809733:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767941.809736:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775767941.809737:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000200:0.0:1775767941.809739:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294254080, portal 25 00000100:00080000:1.0:1775767941.809739:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775767941.809743:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:0.0:1775767941.809745:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775767941.809749:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606124672 : -112966103426944 : ffff994207889880) 00000100:00080000:1.0:1775767941.809749:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5194 00000400:00000200:1.0:1775767941.809758:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000040:0.0:1775767941.809762:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207889880 x1862022294254080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767966 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775767941.809765:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775767941.809770:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775767941.809775:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767941.809778:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767941.809781:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767941.809785:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775767941.809791:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:0.0:1775767941.809792:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000040:1.0:1775767941.809793:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:1.0:1775767941.809797:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775767941.809800:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000100:00000040:0.0:1775767941.809801:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000020:00000001:1.0:1775767941.809802:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:1.0:1775767941.809805:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000400:00000010:0.0:1775767941.809806:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000040:1.0:1775767941.809809:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767941.809812:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:0.0:1775767941.809813:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294254080, offset 0 00000100:00000040:1.0:1775767941.809814:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767941.809818:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775767941.809820:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:1.0:1775767941.809822:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775767941.809825:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767941.809827:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 02000000:00000001:1.0:1775767941.809828:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775767941.809832:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767941.809835:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767941.809836:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775767941.809838:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000400:00000200:0.0:1775767941.809839:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00000001:1.0:1775767941.809842:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767941.809844:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775767941.809848:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff99420788b100. 00000400:00000200:0.0:1775767941.809850:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000020:00000040:1.0:1775767941.809851:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:1.0:1775767941.809855:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775767941.809857:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767941.809858:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:0.0:1775767941.809860:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:1.0:1775767941.809860:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775767941.809863:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767941.809868:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 02000000:00000010:1.0:1775767941.809868:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543d000. 00000100:00000001:1.0:1775767941.809872:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767941.809876:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000100:00100000:1.0:1775767941.809879:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff99420788b100 x1862022294254336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1775767941.809883:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00000001:1.0:1775767941.809890:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775767941.809893:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767941.809894:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225158:225158:256:4294967295] 192.168.202.156@tcp LPNI seq info [225154:225154:8:4294967295] 00000100:00000040:1.0:1775767941.809899:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e26cf977] to pc [ptlrpcd_rcv+-1] req@ffff99420788b100 x1862022294254336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775767941.809909:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767941.809914:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767941.809918:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767941.809924:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775767941.809927:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:0.0:1775767941.809931:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000100:00080000:1.0:1775767941.809931:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775767941.809938:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775767941.809945:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5194 00000800:00000200:0.0:1775767941.809946:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775767941.809953:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000800:00000010:0.0:1775767941.809955:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000400:00000200:1.0:1775767941.809959:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775767941.809963:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775767941.809970:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000800:00000200:0.0:1775767941.809971:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:1.0:1775767941.809972:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767941.809977:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000800:00000200:0.0:1775767941.809979:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000100:00000001:1.0:1775767941.809979:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775767941.809982:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775767941.809984:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:1.0:1775767941.809987:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775767941.809988:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:1.0:1775767941.809991:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:1.0:1775767941.809993:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000800:00000200:0.1:1775767941.809993:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000040:1.0:1775767941.809998:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767941.810000:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:0.1:1775767941.810000:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000100:00000040:1.0:1775767941.810004:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767941.810008:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775767941.810010:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000800:00000100:0.0:1775767941.810014:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000100:00080000:1.0:1775767941.810015:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000800:00000010:0.0:1775767941.810020:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000100:00000001:1.0:1775767941.810020:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767941.810022:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775767941.810026:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1775767941.810029:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000001:1.0:1775767941.810029:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767941.810032:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775767941.810035:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:1.0:1775767941.810039:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767941.810042:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767941.810043:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000100:00000010:1.0:1775767941.810046:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207889f80. 00000020:00000040:1.0:1775767941.810048:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00000400:00000200:0.0:1775767941.810050:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 02000000:00000001:1.0:1775767941.810051:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775767941.810053:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775767941.810056:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:1.0:1775767941.810056:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775767941.810058:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775767941.810061:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767941.810063:0:2399:0:(events.c:34:request_out_callback()) Process entered 02000000:00000010:1.0:1775767941.810064:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543fc00. 00000100:00000001:1.0:1775767941.810068:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1775767941.810070:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207889880 x1862022294254080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767966 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767941.810074:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207889f80 x1862022294254464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775767941.810099:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775767941.810102:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767941.810104:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775767941.810105:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ed2274f4] to pc [ptlrpcd_rcv+-1] req@ffff994207889f80 x1862022294254464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775767941.810112:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207889880 x1862022294254080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767966 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767941.810114:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767941.810117:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767941.810123:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775767941.810124:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775767941.810127:0:8:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775767941.810131:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:0.0:1775767941.810136:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767941.810136:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5194 00000100:00000001:0.0:1775767941.810140:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:1.0:1775767941.810141:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767941.810145:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775767941.810146:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775767941.810148:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775767941.810150:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767941.810150:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767941.810153:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000200:0.0:1775767941.810185:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775767941.810189:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299867966 00000100:00000001:1.0:1775767941.810194:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000010:0.0:1775767941.810197:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942078a3d80. 00000020:00000001:1.0:1775767941.810199:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775767941.810202:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:1.0:1775767941.810205:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:0.0:1775767941.810208:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000020:00000040:1.0:1775767941.810208:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:1.0:1775767941.810213:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000001:0.0:1775767941.810218:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299867966 00000020:00000001:1.0:1775767941.810218:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:1.0:1775767941.810224:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000010:0.0:1775767941.810225:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942078a3a20. 00000100:00000001:1.0:1775767941.810227:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1775767941.810231:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000040:1.0:1775767941.810232:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767941.810236:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775767941.810238:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00000001:0.0:1775767941.810239:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:1.0:1775767941.810241:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775767941.810245:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767941.810246:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767941.810250:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 02000000:00000001:1.0:1775767941.810250:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775767941.810256:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1775767941.810260:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767941/real 1775767941] req@ffff994207889880 x1862022294254080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1775767941.810260:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767941.810264:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775767941.810268:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:1.0:1775767941.810274:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767941.810279:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775767941.810286:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207888700. 00000020:00000040:1.0:1775767941.810291:0:8:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00000400:00000001:0.0:1775767941.810296:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:1.0:1775767941.810296:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775767941.810299:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1775767941.810300:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1775767941.810302:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000010:0.0:1775767941.810304:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 02000000:00000001:1.0:1775767941.810305:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775767941.810308:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767941.810310:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775767941.810313:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 02000000:00000010:1.0:1775767941.810314:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543d400. 00000100:00000001:0.0:1775767941.810318:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1775767941.810320:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1775767941.810323:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207889880 x1862022294254080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767941.810329:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207888700 x1862022294254592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775767941.810349:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000200:0.0:1775767941.810351:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207889880 x1862022294254080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767941.810354:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767941.810363:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009f2ba301] to pc [ptlrpcd_rcv+-1] req@ffff994207888700 x1862022294254592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767941.810365:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775767941.810368:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767941.810375:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767941.810378:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767941.810380:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767941.810387:0:8:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5204) 00000100:00100000:0.0:1775767941.810389:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207889880 x1862022294254080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767941.810398:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775767941.810405:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207889880 x1862022294254080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767966 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767941.810412:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775767941.810416:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767941.810418:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767941.810420:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767941.810423:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775767941.810425:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775767941.810429:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767941.810435:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775767941.810437:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775767941.810443:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207889880 x1862022294254080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767966 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767941.810453:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207889880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294254080:192.168.202.156@tcp:250:kworker.0 00000100:00000001:0.0:1775767941.810476:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767941.810478:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767941.810484:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207889880 x1862022294254080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767966 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767941.810492:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775767941.810495:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775767941.810497:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220029400. 02000000:00000001:0.0:1775767941.810500:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775767941.810502:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775767941.810504:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:0.0:1775767941.810506:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775767941.810509:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543d800. 02000000:00000001:0.0:1775767941.810514:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775767941.810516:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775767941.810518:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207889880. 00000100:00000001:0.0:1775767941.810521:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775767941.810523:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767941.810525:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767941.810532:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767941.810535:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767941.810537:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775767941.810539:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767941.810541:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775767941.810545:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207888e00 x1862022294254208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767941.810553:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767941.810556:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767941.810559:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775767941.810561:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775767941.810564:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767941.810568:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207888e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294254208:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775767941.810573:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775767941.810575:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775767941.810577:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767941.810580:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775767941.810584:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220029400. 02000000:00000001:0.0:1775767941.810587:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767941.810593:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775767941.810596:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775767941.810600:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775767941.810603:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294254208, portal 10 00000100:00000001:0.0:1775767941.810605:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775767941.810606:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606121984 : -112966103429632 : ffff994207888e00) 00000100:00000040:0.0:1775767941.810612:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207888e00 x1862022294254208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767966 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767941.810619:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775767941.810621:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775767941.810624:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775767941.810626:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294254208, offset 0 00000400:00000200:0.0:1775767941.810630:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775767941.810635:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767941.810639:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775767941.810642:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775767941.810645:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775767941.810647:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775767941.810650:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775767941.810653:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225159:225159:256:4294967295] 192.168.202.156@tcp LPNI seq info [225155:225155:8:4294967295] 00000400:00000200:0.0:1775767941.810657:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767941.810664:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775767941.810668:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767941.810672:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775767941.810676:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775767941.810679:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775767941.810684:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767941.810687:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1775767941.810691:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767941.810694:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775767941.810697:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775767941.810700:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775767941.810703:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775767941.810705:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775767941.810708:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775767941.810712:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207888e00 x1862022294254208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767966 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767941.810719:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767941.810722:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207888e00 x1862022294254208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767966 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767941.810729:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767941.810731:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775767941.810734:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767941.810738:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:0.0:1775767941.810742:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775767941.810744:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299866643 00000100:00000001:0.0:1775767941.810747:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299867966 00000100:00000001:0.0:1775767941.810749:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299866643 00000100:00000001:0.0:1775767941.810752:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299867966 00000100:00000001:0.0:1775767941.810754:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775767941.810756:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767941.810758:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775767941.810762:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767941/real 1775767941] req@ffff994207888e00 x1862022294254208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775767941.810770:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775767941.810771:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775767941.810773:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775767941.810776:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775767941.810777:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775767941.810779:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775767941.810783:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207888e00 x1862022294254208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775767941.810791:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207888e00 x1862022294254208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767941.810797:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775767941.810799:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767941.810800:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767941.810802:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775767941.810806:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207888e00 x1862022294254208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767941.810812:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775767941.810816:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207888e00 x1862022294254208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767966 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767941.810823:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775767941.810825:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767941.810827:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767941.810828:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767941.810830:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775767941.810833:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775767941.810835:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767941.810839:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775767941.810841:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775767941.810846:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207888e00 x1862022294254208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767966 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767941.810854:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207888e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294254208:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775767941.810858:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775767941.810862:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff99420788b100 x1862022294254336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767941.810868:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767941.810870:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767941.810872:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775767941.810874:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775767941.810876:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767941.810879:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff99420788b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294254336:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775767941.810882:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775767941.810884:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775767941.810885:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767941.810887:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775767941.810891:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002b800. 02000000:00000001:0.0:1775767941.810893:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767941.810896:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775767941.810899:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775767941.810902:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775767941.810905:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294254336, portal 10 00000100:00000001:0.0:1775767941.810907:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775767941.810908:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606130944 : -112966103420672 : ffff99420788b100) 00000100:00000040:0.0:1775767941.810914:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff99420788b100 x1862022294254336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767966 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767941.810921:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775767941.810923:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775767941.810925:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775767941.810928:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294254336, offset 0 00000400:00000200:0.0:1775767941.810932:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775767941.810935:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767941.810938:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775767941.810941:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775767941.810944:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775767941.810946:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775767941.810948:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775767941.810951:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225160:225160:256:4294967295] 192.168.202.156@tcp LPNI seq info [225156:225156:8:4294967295] 00000400:00000200:0.0:1775767941.810954:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767941.810960:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775767941.810964:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767941.810967:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775767941.810970:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775767941.810973:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775767941.810977:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767941.810980:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1775767941.810984:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767941.810986:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775767941.810989:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775767941.810992:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775767941.810995:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775767941.810997:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775767941.811000:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775767941.811003:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff99420788b100 x1862022294254336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767966 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767941.811012:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767941.811015:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff99420788b100 x1862022294254336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767966 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767941.811021:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767941.811023:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775767941.811026:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767941.811030:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:0.0:1775767941.811034:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775767941.811036:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299866635 00000100:00000001:0.0:1775767941.811038:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299867966 00000100:00000001:0.0:1775767941.811040:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299866635 00000100:00000001:0.0:1775767941.811043:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299867966 00000100:00000001:0.0:1775767941.811045:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775767941.811047:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767941.811049:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775767941.811053:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767941/real 1775767941] req@ffff99420788b100 x1862022294254336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775767941.811060:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775767941.811062:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775767941.811064:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775767941.811066:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775767941.811068:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775767941.811070:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775767941.811073:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff99420788b100 x1862022294254336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775767941.811081:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff99420788b100 x1862022294254336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767941.811088:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775767941.811089:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767941.811091:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767941.811093:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775767941.811096:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff99420788b100 x1862022294254336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767941.811104:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775767941.811107:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff99420788b100 x1862022294254336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767966 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767941.811114:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775767941.811116:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767941.811117:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767941.811119:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767941.811121:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775767941.811125:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775767941.811127:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767941.811131:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775767941.811134:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775767941.811138:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff99420788b100 x1862022294254336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767966 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767941.811146:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff99420788b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294254336:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775767941.811150:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775767941.811179:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207889f80 x1862022294254464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767941.811187:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767941.811189:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767941.811191:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775767941.811194:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775767941.811196:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767941.811199:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207889f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294254464:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775767941.811203:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775767941.811205:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775767941.811206:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767941.811208:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775767941.811211:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220028000. 02000000:00000001:0.0:1775767941.811213:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767941.811217:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775767941.811220:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775767941.811223:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775767941.811226:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294254464, portal 4 00000100:00000001:0.0:1775767941.811228:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775767941.811230:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606126464 : -112966103425152 : ffff994207889f80) 00000100:00000040:0.0:1775767941.811236:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207889f80 x1862022294254464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767966 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767941.811243:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775767941.811245:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775767941.811247:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775767941.811250:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294254464, offset 0 00000400:00000200:0.0:1775767941.811254:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775767941.811258:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767941.811262:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775767941.811265:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775767941.811268:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775767941.811270:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775767941.811273:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775767941.811276:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225161:225161:256:4294967295] 192.168.202.156@tcp LPNI seq info [225157:225157:8:4294967295] 00000400:00000200:0.0:1775767941.811281:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767941.811288:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775767941.811293:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767941.811296:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775767941.811301:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775767941.811304:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775767941.811307:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767941.811310:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1775767941.811314:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767941.811316:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775767941.811319:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775767941.811323:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775767941.811326:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775767941.811327:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775767941.811330:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775767941.811333:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207889f80 x1862022294254464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767966 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767941.811341:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767941.811344:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207889f80 x1862022294254464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767966 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767941.811357:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767941.811358:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775767941.811361:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767941.811364:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:0.0:1775767941.811367:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775767941.811369:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299866659 00000100:00000001:0.0:1775767941.811372:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299867966 00000100:00000001:0.0:1775767941.811374:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299866659 00000100:00000001:0.0:1775767941.811377:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299867966 00000100:00000001:0.0:1775767941.811379:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775767941.811381:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767941.811383:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775767941.811387:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767941/real 1775767941] req@ffff994207889f80 x1862022294254464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775767941.811395:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775767941.811396:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775767941.811398:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775767941.811400:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775767941.811402:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775767941.811404:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775767941.811408:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207889f80 x1862022294254464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775767941.811417:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207889f80 x1862022294254464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767941.811425:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775767941.811426:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767941.811428:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767941.811429:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775767941.811432:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207889f80 x1862022294254464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767941.811439:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775767941.811444:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207889f80 x1862022294254464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767966 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767941.811450:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775767941.811453:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767941.811455:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767941.811473:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767941.811475:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775767941.811479:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775767941.811482:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767941.811486:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775767941.811488:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775767941.811491:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207889f80 x1862022294254464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767966 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767941.811499:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207889f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294254464:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775767941.811504:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775767941.811507:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207888700 x1862022294254592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767941.811514:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767941.811515:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767941.811518:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775767941.811519:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775767941.811522:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767941.811525:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207888700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294254592:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775767941.811528:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775767941.811530:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775767941.811532:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767941.811534:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775767941.811537:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002a800. 02000000:00000001:0.0:1775767941.811539:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767941.811543:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775767941.811546:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775767941.811549:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775767941.811552:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294254592, portal 4 00000100:00000001:0.0:1775767941.811554:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775767941.811555:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606120192 : -112966103431424 : ffff994207888700) 00000100:00000040:0.0:1775767941.811560:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207888700 x1862022294254592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767966 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767941.811567:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775767941.811569:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775767941.811571:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775767941.811573:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294254592, offset 0 00000400:00000200:0.0:1775767941.811578:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775767941.811582:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767941.811585:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775767941.811588:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775767941.811591:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775767941.811593:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775767941.811596:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775767941.811599:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225162:225162:256:4294967295] 192.168.202.156@tcp LPNI seq info [225158:225158:8:4294967295] 00000400:00000200:0.0:1775767941.811603:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767941.811610:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775767941.811614:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767941.811617:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775767941.811621:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775767941.811624:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775767941.811629:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767941.811631:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1775767941.811635:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767941.811638:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775767941.811641:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775767941.811644:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775767941.811647:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775767941.811649:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775767941.811651:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775767941.811655:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207888700 x1862022294254592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767966 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767941.811663:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767941.811666:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207888700 x1862022294254592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767966 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767941.811672:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767941.811673:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775767941.811676:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767941.811678:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:0.0:1775767941.811681:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775767941.811683:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299866662 00000100:00000001:0.0:1775767941.811685:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299867966 00000100:00000001:0.0:1775767941.811687:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299866662 00000100:00000001:0.0:1775767941.811690:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299867966 00000100:00000001:0.0:1775767941.811692:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775767941.811693:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767941.811695:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775767941.811698:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767941/real 1775767941] req@ffff994207888700 x1862022294254592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775767941.811705:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775767941.811707:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775767941.811708:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775767941.811711:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775767941.811713:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775767941.811716:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775767941.811719:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207888700 x1862022294254592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775767941.811727:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207888700 x1862022294254592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767941.811734:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775767941.811735:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767941.811738:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767941.811739:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775767941.811743:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207888700 x1862022294254592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767941.811750:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775767941.811754:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207888700 x1862022294254592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767966 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767941.811761:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775767941.811764:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767941.811765:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767941.811767:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767941.811769:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775767941.811772:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775767941.811775:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767941.811779:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775767941.811781:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775767941.811785:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207888700 x1862022294254592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767966 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767941.811793:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207888700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294254592:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775767941.811797:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767941.811799:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767941.811802:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207888e00 x1862022294254208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767966 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767941.811809:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775767941.811811:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775767941.811813:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220029400. 02000000:00000001:0.0:1775767941.811815:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775767941.811817:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775767941.811818:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:0.0:1775767941.811821:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775767941.811823:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543e400. 02000000:00000001:0.0:1775767941.811825:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775767941.811827:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775767941.811828:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207888e00. 00000100:00000001:0.0:1775767941.811831:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775767941.811833:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767941.811835:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767941.811838:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff99420788b100 x1862022294254336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767966 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767941.811844:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775767941.811845:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775767941.811847:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002b800. 02000000:00000001:0.0:1775767941.811850:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775767941.811851:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775767941.811853:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:0.0:1775767941.811854:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775767941.811856:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543d000. 02000000:00000001:0.0:1775767941.811858:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775767941.811859:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775767941.811861:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff99420788b100. 00000100:00000001:0.0:1775767941.811863:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775767941.811865:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767941.811866:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767941.811870:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207889f80 x1862022294254464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767966 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767941.811877:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775767941.811878:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775767941.811880:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220028000. 02000000:00000001:0.0:1775767941.811882:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775767941.811884:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775767941.811885:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:0.0:1775767941.811887:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775767941.811888:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543fc00. 02000000:00000001:0.0:1775767941.811891:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775767941.811892:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775767941.811894:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207889f80. 00000100:00000001:0.0:1775767941.811896:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775767941.811898:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767941.811899:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767941.811902:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207888700 x1862022294254592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767966 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767941.811909:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775767941.811910:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775767941.811912:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002a800. 02000000:00000001:0.0:1775767941.811914:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775767941.811916:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775767941.811917:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:0.0:1775767941.811919:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775767941.811921:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543d400. 02000000:00000001:0.0:1775767941.811922:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775767941.811924:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775767941.811925:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207888700. 00000100:00000001:0.0:1775767941.811927:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775767941.811929:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767941.811930:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767941.811934:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767941.811936:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767941.811937:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775767941.811939:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767941.873239:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767941.873243:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767941.873247:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767941.873249:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767941.873252:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767941.873253:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767941.873255:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767941.873256:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767941.873258:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767941.873259:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767941.873274:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767941.873276:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767941.873278:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767941.937258:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767941.937264:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767941.937266:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767941.937270:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767941.937273:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767941.937276:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767941.937279:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767941.937281:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767941.937283:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767942.833254:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767942.833255:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767942.833255:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767942.833262:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767942.833264:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767942.833264:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767942.833264:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767942.833267:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767942.833268:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767942.833271:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:0.0:1775767942.833274:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767942.833275:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1775767942.833275:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775767942.833281:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767942.833282:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767942.833282:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767942.833284:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767942.833286:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767942.833287:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767942.833289:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767942.833293:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767942.833295:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767942.833299:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767942.833302:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767942.833304:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767942.833308:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767942.833311:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767942.897291:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767942.897300:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767942.897309:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767942.897311:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767942.897315:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767942.897316:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767942.897320:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767942.897322:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767942.897326:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767942.897327:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767942.897353:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767942.897357:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767942.897360:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767942.961251:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767942.961258:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767942.961260:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767942.961266:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767942.961269:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767942.961273:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767942.961275:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767942.961278:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767942.961280:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767943.153296:0:14953:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4299867966, at 4299867968 00000100:00000040:1.0:1775767943.153296:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299867966, at 4299867968 00000100:00000010:1.0:1775767943.153314:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3a20. 00000100:00000010:0.0:1775767943.153319:0:14953:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3d80. 00000100:00000001:0.0:1775767943.857253:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767943.857253:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767943.857253:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767943.857261:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767943.857262:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767943.857262:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767943.857264:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767943.857266:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767943.857267:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767943.857271:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:2.0:1775767943.857273:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775767943.857273:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775767943.857279:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767943.857279:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767943.857279:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767943.857285:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767943.857285:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767943.857286:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767943.857291:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767943.857294:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767943.857296:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767943.857298:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767943.857301:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767943.857303:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767943.857304:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767943.857308:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767943.857312:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767943.921285:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767943.921291:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767943.921296:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767943.921297:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767943.921301:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767943.921302:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767943.921304:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767943.921305:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767943.921308:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767943.921309:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767943.921326:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767943.921329:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767943.921331:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767943.985269:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767943.985280:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767943.985282:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767943.985289:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767943.985292:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767943.985297:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767943.985299:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767943.985302:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767943.985313:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767944.881246:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767944.881250:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767944.881255:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767944.881255:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767944.881258:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767944.881258:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767944.881267:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775767944.881268:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767944.881274:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767944.881275:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767944.881276:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767944.881279:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767944.881280:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767944.881284:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767944.881289:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767944.881289:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767944.881292:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767944.881298:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767944.881299:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767944.881299:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767944.881304:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767944.881306:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767944.881307:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767944.881312:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767944.881316:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767944.881319:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767944.881322:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767944.945313:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767944.945318:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767944.945323:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767944.945324:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767944.945327:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767944.945328:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767944.945331:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767944.945333:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767944.945336:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767944.945337:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767944.945357:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767944.945359:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767944.945362:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767945.009260:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767945.009265:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767945.009266:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767945.009271:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767945.009274:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767945.009277:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767945.009279:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767945.009281:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767945.009283:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767945.905246:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767945.905247:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767945.905252:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767945.905254:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767945.905254:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767945.905256:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767945.905257:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767945.905257:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767945.905259:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767945.905265:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:0.0:1775767945.905266:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775767945.905266:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775767945.905272:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767945.905273:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767945.905273:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767945.905277:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767945.905278:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767945.905278:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767945.905284:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767945.905287:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767945.905289:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767945.905292:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767945.905294:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767945.905297:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767945.905298:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767945.905300:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767945.905304:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767945.969265:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767945.969275:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767945.969283:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767945.969297:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775767945.969299:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775767945.969304:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767945.969307:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775767945.969310:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767945.969313:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767945.969315:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767945.969321:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767945.969323:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775767945.969324:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775767945.969326:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767945.969328:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775767945.969329:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767945.969331:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767945.969333:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767945.969337:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767945.969339:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775767945.969340:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775767945.969342:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767945.969344:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775767945.969345:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767945.969347:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767945.969349:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767945.969353:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767945.969355:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775767945.969356:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775767945.969358:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767945.969360:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775767945.969361:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767945.969363:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767945.969365:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767945.969391:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767945.969394:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767945.969397:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767946.033298:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767946.033307:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767946.033309:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767946.033317:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767946.033322:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767946.033328:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767946.033331:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767946.033335:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767946.033339:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767946.929277:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767946.929278:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767946.929278:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767946.929290:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767946.929290:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767946.929291:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767946.929300:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767946.929300:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767946.929301:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775767946.929314:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775767946.929317:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775767946.929317:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:1.0:1775767946.929318:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767946.929330:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767946.929332:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1775767946.929334:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775767946.929335:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767946.929335:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767946.929337:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767946.929343:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1775767946.929343:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767946.929344:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767946.929352:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1775767946.929353:0:8:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775767946.929356:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767946.929357:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:0.0:1775767946.929358:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767946.929364:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1775767946.929368:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5199 00000100:00000001:0.0:1775767946.929370:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767946.929370:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767946.929375:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1775767946.929381:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767946.929381:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767946.929392:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775767946.929395:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775767946.929400:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767946.929403:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767946.929405:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767946.929419:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775767946.929423:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775767946.929425:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:2.0:1775767946.929427:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775767946.929431:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:2.0:1775767946.929434:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:2.0:1775767946.929436:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:2.0:1775767946.929442:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767946.929445:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767946.929447:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767946.929448:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775767946.929451:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:2.0:1775767946.929453:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775767946.929455:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767946.929457:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775767946.929460:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767946.929462:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767946.929463:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775767946.929466:0:8:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775767946.929476:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dff480. 00000020:00000040:2.0:1775767946.929479:0:8:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:2.0:1775767946.929482:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775767946.929484:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767946.929487:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775767946.929490:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775767946.929495:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775767946.929503:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1e400. 00000100:00000001:2.0:1775767946.929510:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767946.929519:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dff480 x1862022294254720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775767946.929527:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775767946.929532:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767946.929536:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093a15639] to pc [ptlrpcd_rcv+-1] req@ffff994207dff480 x1862022294254720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767946.929554:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775767946.929556:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767946.929559:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775767946.929560:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775767946.929561:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775767946.929564:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775767946.929566:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5199 00000400:00000200:2.0:1775767946.929568:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767946.929571:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775767946.929573:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775767946.929576:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767946.929578:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767946.929579:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767946.929581:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775767946.929582:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775767946.929583:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000100:00000001:0.0:1775767946.929584:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1775767946.929585:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:0.0:1775767946.929588:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000040:2.0:1775767946.929588:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000100:00000001:0.0:1775767946.929592:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000001:2.0:1775767946.929592:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:2.0:1775767946.929596:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:2.0:1775767946.929601:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000040:0.0:1775767946.929602:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dff480 x1862022294254720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767946.929603:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767946.929607:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767946.929611:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775767946.929617:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00000001:0.0:1775767946.929619:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767946.929622:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775767946.929622:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775767946.929626:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767946.929628:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767946.929628:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775767946.929632:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775767946.929635:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767946.929635:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767946.929639:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767946.929640:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294254720:192.168.202.156@tcp:250:kworker.0 00000100:00000001:2.0:1775767946.929641:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775767946.929646:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000008:00100000:2.0:1775767946.929647:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1775767946.929650:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000008:00000001:2.0:1775767946.929650:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767946.929651:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767946.929653:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775767946.929655:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfce00. 02000000:00000001:0.0:1775767946.929657:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:2.0:1775767946.929658:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:2.0:1775767946.929660:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775767946.929661:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767946.929662:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000010:0.0:1775767946.929664:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002b800. 02000000:00000001:2.0:1775767946.929664:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775767946.929667:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767946.929667:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775767946.929674:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1f800. 00000400:00000200:0.0:1775767946.929678:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767946.929680:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775767946.929684:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000100:00100000:2.0:1775767946.929685:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfce00 x1862022294254848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775767946.929691:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000010:0.0:1775767946.929692:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000001:2.0:1775767946.929694:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767946.929697:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e5a7debe] to pc [ptlrpcd_rcv+-1] req@ffff994207dfce00 x1862022294254848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775767946.929701:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294254720, portal 25 00000100:00000001:0.0:1775767946.929706:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767946.929707:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767946.929709:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611849856 : -112966097701760 : ffff994207dff480) 00000100:00080000:2.0:1775767946.929710:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767946.929715:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775767946.929716:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775767946.929718:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:0.0:1775767946.929719:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dff480 x1862022294254720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767971 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767946.929722:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775767946.929728:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5199 00000400:00000200:2.0:1775767946.929732:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767946.929733:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775767946.929735:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775767946.929737:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767946.929741:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767946.929744:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775767946.929745:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1775767946.929746:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775767946.929748:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000040:0.0:1775767946.929751:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000020:00000040:2.0:1775767946.929751:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:2.0:1775767946.929753:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000010:0.0:1775767946.929755:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000020:00000040:2.0:1775767946.929755:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:2.0:1775767946.929759:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000200:0.0:1775767946.929762:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294254720, offset 0 00000020:00000001:2.0:1775767946.929762:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:2.0:1775767946.929768:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767946.929770:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1775767946.929774:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00000040:2.0:1775767946.929777:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767946.929780:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775767946.929783:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000400:00000200:0.0:1775767946.929786:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767946.929793:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775767946.929800:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:2.0:1775767946.929800:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775767946.929805:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767946.929807:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 02000000:00000001:2.0:1775767946.929810:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:0.0:1775767946.929814:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 02000000:00000001:2.0:1775767946.929816:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1775767946.929817:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 02000000:00000001:2.0:1775767946.929822:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767946.929824:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225163:225163:256:4294967295] 192.168.202.156@tcp LPNI seq info [225159:225159:8:4294967295] 00000400:00000200:0.0:1775767946.929827:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767946.929832:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775767946.929836:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000400:00000200:0.0:1775767946.929842:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000008:00000001:2.0:1775767946.929843:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767946.929848:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1775767946.929852:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00000010:2.0:1775767946.929854:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfed80. 00000020:00000040:2.0:1775767946.929857:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000800:00000010:0.0:1775767946.929858:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 02000000:00000001:2.0:1775767946.929860:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775767946.929862:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767946.929863:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775767946.929865:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:0.0:1775767946.929868:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 02000000:00000001:2.0:1775767946.929868:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1775767946.929874:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 02000000:00000010:2.0:1775767946.929875:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1d000. 00000100:00000001:2.0:1775767946.929881:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775767946.929884:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767946.929888:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000100:00100000:2.0:1775767946.929889:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfed80 x1862022294254976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767946.929900:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767946.929904:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000100:00000001:2.0:1775767946.929904:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775767946.929908:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767946.929912:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:2.0:1775767946.929912:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb9b9eae] to pc [ptlrpcd_rcv+-1] req@ffff994207dfed80 x1862022294254976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775767946.929923:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000100:00000001:2.0:1775767946.929924:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775767946.929928:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1775767946.929929:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775767946.929933:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00080000:2.0:1775767946.929935:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775767946.929938:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1775767946.929938:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775767946.929942:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:0.0:1775767946.929944:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dff480 x1862022294254720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767971 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767946.929948:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775767946.929956:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5199 00000100:00000001:0.0:1775767946.929957:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767946.929963:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dff480 x1862022294254720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767971 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1775767946.929964:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767946.929972:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775767946.929978:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1775767946.929980:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767946.929985:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:2.0:1775767946.929985:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767946.929990:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767946.929993:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775767946.929994:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767946.929996:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775767946.930000:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000400:00000200:0.0:1775767946.930003:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000040:2.0:1775767946.930003:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:2.0:1775767946.930007:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:0.0:1775767946.930009:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299873085 00000020:00000040:2.0:1775767946.930011:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:2.0:1775767946.930015:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:2.0:1775767946.930016:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000010:0.0:1775767946.930018:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942078a3d80. 00000100:00000040:2.0:1775767946.930019:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767946.930024:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767946.930025:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767946.930027:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775767946.930028:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:2.0:1775767946.930030:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775767946.930031:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1775767946.930033:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767946.930035:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775767946.930038:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775767946.930039:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299873085 02000000:00000001:2.0:1775767946.930040:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767946.930045:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000010:0.0:1775767946.930046:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942078a3a80. 00000008:00100000:2.0:1775767946.930047:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000100:00000001:0.0:1775767946.930051:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000008:00000001:2.0:1775767946.930051:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767946.930055:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767946.930057:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000010:2.0:1775767946.930059:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfe300. 00000100:00000001:0.0:1775767946.930060:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775767946.930063:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00000100:00000001:0.0:1775767946.930065:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 02000000:00000001:2.0:1775767946.930068:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775767946.930070:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767946.930073:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767946/real 1775767946] req@ffff994207dff480 x1862022294254720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767946.930074:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775767946.930076:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775767946.930079:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775767946.930087:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1f400. 00000400:00000001:0.0:1775767946.930089:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775767946.930093:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1775767946.930093:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775767946.930097:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775767946.930102:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000100:00100000:2.0:1775767946.930102:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfe300 x1862022294255104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:0.0:1775767946.930107:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775767946.930113:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1775767946.930118:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000200:0.0:1775767946.930122:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dff480 x1862022294254720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767946.930122:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767946.930132:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c661fef] to pc [ptlrpcd_rcv+-1] req@ffff994207dfe300 x1862022294255104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775767946.930145:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dff480 x1862022294254720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767946.930148:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775767946.930178:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767946.930183:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1775767946.930185:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775767946.930188:0:8:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775767946.930190:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767946.930193:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:0.0:1775767946.930194:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767946.930198:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:2.0:1775767946.930199:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5199 00000100:00000001:0.0:1775767946.930201:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1775767946.930208:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00100000:0.0:1775767946.930209:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dff480 x1862022294254720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1775767946.930217:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:0.0:1775767946.930221:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1775767946.930222:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775767946.930227:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000040:0.0:1775767946.930229:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dff480 x1862022294254720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767971 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767946.930232:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767946.930238:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775767946.930241:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767946.930241:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:0.0:1775767946.930246:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1775767946.930246:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:0.0:1775767946.930249:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000040:2.0:1775767946.930249:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000100:00000001:0.0:1775767946.930253:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1775767946.930253:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775767946.930256:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000100:00000001:0.0:1775767946.930257:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000020:00000001:2.0:1775767946.930260:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000001:0.0:1775767946.930261:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:2.0:1775767946.930263:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00080000:0.0:1775767946.930267:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000040:2.0:1775767946.930268:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767946.930272:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767946.930275:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767946.930277:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00080000:0.0:1775767946.930278:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000020:00000040:2.0:1775767946.930279:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00000001:0.0:1775767946.930284:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00080000:2.0:1775767946.930286:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775767946.930291:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767946.930295:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dff480 x1862022294254720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767971 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1775767946.930295:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775767946.930300:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767946.930306:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767946.930311:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775767946.930314:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000100:00100000:0.0:1775767946.930315:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294254720:192.168.202.156@tcp:250:kworker.0 00000008:00000001:2.0:1775767946.930319:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767946.930323:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767946.930331:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:2.0:1775767946.930331:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfc700. 00000020:00000040:2.0:1775767946.930337:0:8:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00000100:00000001:0.0:1775767946.930338:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1775767946.930342:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000040:0.0:1775767946.930344:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dff480 x1862022294254720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767971 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1775767946.930345:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767946.930350:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775767946.930353:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775767946.930356:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767946.930363:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000010:2.0:1775767946.930364:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1fc00. 02000000:00000001:0.0:1775767946.930367:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1775767946.930370:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775767946.930372:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002b800. 02000000:00000001:0.0:1775767946.930377:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00100000:2.0:1775767946.930378:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfc700 x1862022294255232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000001:0.0:1775767946.930381:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775767946.930384:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:0.0:1775767946.930389:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775767946.930392:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1e400. 00000100:00000001:2.0:1775767946.930395:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:0.0:1775767946.930398:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:2.0:1775767946.930399:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767946.930401:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775767946.930405:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dff480. 00000100:00000040:2.0:1775767946.930419:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a176dd1] to pc [ptlrpcd_rcv+-1] req@ffff994207dfc700 x1862022294255232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767946.930422:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775767946.930424:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767946.930430:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767946.930439:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767946.930442:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767946.930445:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767946.930445:0:8:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5209) 00000100:00000001:0.0:1775767946.930449:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775767946.930453:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767946.930455:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775767946.930461:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfce00 x1862022294254848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767946.930470:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767946.930471:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767946.930474:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775767946.930476:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775767946.930478:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767946.930482:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294254848:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775767946.930486:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775767946.930487:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775767946.930489:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767946.930490:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775767946.930493:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002b800. 02000000:00000001:0.0:1775767946.930495:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767946.930498:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775767946.930500:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775767946.930503:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775767946.930505:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294254848, portal 10 00000100:00000001:0.0:1775767946.930507:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775767946.930508:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611840000 : -112966097711616 : ffff994207dfce00) 00000100:00000040:0.0:1775767946.930513:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfce00 x1862022294254848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767971 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767946.930518:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775767946.930520:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775767946.930521:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775767946.930523:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294254848, offset 0 00000400:00000200:0.0:1775767946.930526:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775767946.930529:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767946.930531:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775767946.930534:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775767946.930535:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775767946.930537:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775767946.930539:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775767946.930541:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225164:225164:256:4294967295] 192.168.202.156@tcp LPNI seq info [225160:225160:8:4294967295] 00000400:00000200:0.0:1775767946.930544:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767946.930549:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775767946.930552:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767946.930554:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775767946.930558:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775767946.930560:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775767946.930563:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767946.930565:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1775767946.930568:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767946.930570:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775767946.930573:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775767946.930575:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775767946.930578:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775767946.930579:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775767946.930580:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775767946.930583:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfce00 x1862022294254848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767971 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767946.930588:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767946.930590:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfce00 x1862022294254848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767971 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767946.930595:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767946.930596:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775767946.930598:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767946.930601:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:0.0:1775767946.930603:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775767946.930604:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299871763 00000100:00000001:0.0:1775767946.930606:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299873085 00000100:00000001:0.0:1775767946.930608:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299871763 00000100:00000001:0.0:1775767946.930610:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299873085 00000100:00000001:0.0:1775767946.930612:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775767946.930613:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767946.930614:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775767946.930616:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767946/real 1775767946] req@ffff994207dfce00 x1862022294254848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775767946.930621:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775767946.930622:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775767946.930623:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775767946.930624:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775767946.930625:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775767946.930627:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775767946.930629:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfce00 x1862022294254848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775767946.930635:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfce00 x1862022294254848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767946.930639:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775767946.930640:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767946.930641:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767946.930642:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775767946.930644:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfce00 x1862022294254848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767946.930648:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775767946.930650:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfce00 x1862022294254848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767971 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767946.930654:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775767946.930656:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767946.930657:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767946.930658:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767946.930659:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775767946.930661:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775767946.930662:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767946.930664:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775767946.930666:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775767946.930668:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfce00 x1862022294254848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767971 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767946.930673:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294254848:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775767946.930675:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775767946.930677:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfed80 x1862022294254976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767946.930686:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767946.930687:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767946.930689:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775767946.930690:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775767946.930691:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767946.930693:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294254976:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775767946.930695:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775767946.930696:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775767946.930697:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767946.930698:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775767946.930700:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220029400. 02000000:00000001:0.0:1775767946.930701:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767946.930703:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775767946.930705:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775767946.930706:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775767946.930708:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294254976, portal 10 00000100:00000001:0.0:1775767946.930709:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775767946.930710:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611848064 : -112966097703552 : ffff994207dfed80) 00000100:00000040:0.0:1775767946.930714:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfed80 x1862022294254976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767971 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767946.930719:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775767946.930720:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775767946.930722:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775767946.930723:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294254976, offset 0 00000400:00000200:0.0:1775767946.930726:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775767946.930728:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767946.930730:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775767946.930732:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775767946.930733:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775767946.930734:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775767946.930736:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775767946.930739:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225165:225165:256:4294967295] 192.168.202.156@tcp LPNI seq info [225161:225161:8:4294967295] 00000400:00000200:0.0:1775767946.930742:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767946.930746:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775767946.930749:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767946.930751:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775767946.930753:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775767946.930754:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775767946.930757:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767946.930759:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1775767946.930761:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767946.930763:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775767946.930765:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775767946.930767:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775767946.930769:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775767946.930770:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775767946.930772:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775767946.930774:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfed80 x1862022294254976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767971 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767946.930779:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767946.930781:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfed80 x1862022294254976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767971 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767946.930785:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767946.930786:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775767946.930788:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767946.930790:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:0.0:1775767946.930792:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775767946.930793:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299871755 00000100:00000001:0.0:1775767946.930795:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299873085 00000100:00000001:0.0:1775767946.930797:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299871755 00000100:00000001:0.0:1775767946.930798:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299873085 00000100:00000001:0.0:1775767946.930800:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775767946.930802:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767946.930803:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775767946.930805:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767946/real 1775767946] req@ffff994207dfed80 x1862022294254976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775767946.930811:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775767946.930811:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775767946.930812:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775767946.930813:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775767946.930814:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775767946.930816:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775767946.930817:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfed80 x1862022294254976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775767946.930822:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfed80 x1862022294254976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767946.930827:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775767946.930828:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767946.930829:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767946.930830:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775767946.930832:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfed80 x1862022294254976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767946.930836:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775767946.930838:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfed80 x1862022294254976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767971 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767946.930842:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775767946.930843:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767946.930844:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767946.930845:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767946.930846:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775767946.930848:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775767946.930850:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767946.930852:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775767946.930854:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775767946.930856:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfed80 x1862022294254976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767971 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767946.930861:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294254976:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775767946.930863:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775767946.930865:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfe300 x1862022294255104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767946.930869:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767946.930870:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767946.930871:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775767946.930872:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775767946.930873:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767946.930875:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294255104:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775767946.930877:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775767946.930878:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775767946.930879:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767946.930880:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775767946.930882:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002bc00. 02000000:00000001:0.0:1775767946.930883:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767946.930885:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775767946.930886:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775767946.930888:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775767946.930890:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294255104, portal 4 00000100:00000001:0.0:1775767946.930891:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775767946.930892:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611845376 : -112966097706240 : ffff994207dfe300) 00000100:00000040:0.0:1775767946.930896:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfe300 x1862022294255104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767971 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767946.930901:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775767946.930903:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775767946.930904:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775767946.930906:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294255104, offset 0 00000400:00000200:0.0:1775767946.930909:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775767946.930911:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767946.930913:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775767946.930915:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775767946.930917:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775767946.930918:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775767946.930920:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775767946.930922:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225166:225166:256:4294967295] 192.168.202.156@tcp LPNI seq info [225162:225162:8:4294967295] 00000400:00000200:0.0:1775767946.930924:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767946.930928:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775767946.930931:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767946.930933:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775767946.930936:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775767946.930937:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775767946.930940:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767946.930942:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1775767946.930944:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767946.930946:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775767946.930948:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775767946.930950:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775767946.930951:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775767946.930952:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775767946.930954:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775767946.930956:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfe300 x1862022294255104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767971 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767946.930961:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767946.930963:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfe300 x1862022294255104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767971 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767946.930968:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767946.930969:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775767946.930972:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767946.930975:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:0.0:1775767946.930979:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775767946.930981:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299871778 00000100:00000001:0.0:1775767946.930983:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299873085 00000100:00000001:0.0:1775767946.930985:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299871778 00000100:00000001:0.0:1775767946.930988:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299873085 00000100:00000001:0.0:1775767946.930990:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775767946.930992:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767946.930994:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775767946.930998:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767946/real 1775767946] req@ffff994207dfe300 x1862022294255104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775767946.931008:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775767946.931009:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775767946.931011:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775767946.931014:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775767946.931016:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775767946.931018:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775767946.931021:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfe300 x1862022294255104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775767946.931031:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfe300 x1862022294255104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767946.931036:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775767946.931037:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767946.931038:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767946.931039:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775767946.931042:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfe300 x1862022294255104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767946.931046:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775767946.931049:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfe300 x1862022294255104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767971 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767946.931054:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775767946.931056:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767946.931057:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767946.931058:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767946.931059:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775767946.931063:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775767946.931064:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767946.931066:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775767946.931067:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775767946.931069:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfe300 x1862022294255104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767971 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767946.931074:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294255104:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775767946.931077:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775767946.931079:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfc700 x1862022294255232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767946.931083:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767946.931085:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767946.931087:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775767946.931088:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775767946.931090:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767946.931093:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294255232:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775767946.931097:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775767946.931099:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775767946.931100:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767946.931102:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775767946.931106:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220028400. 02000000:00000001:0.0:1775767946.931108:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767946.931112:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775767946.931115:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775767946.931118:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775767946.931121:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294255232, portal 4 00000100:00000001:0.0:1775767946.931124:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775767946.931125:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611838208 : -112966097713408 : ffff994207dfc700) 00000100:00000040:0.0:1775767946.931132:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfc700 x1862022294255232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767971 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767946.931140:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775767946.931143:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775767946.931145:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775767946.931147:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294255232, offset 0 00000400:00000200:0.0:1775767946.931149:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775767946.931152:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767946.931171:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775767946.931173:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775767946.931175:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775767946.931177:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775767946.931179:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775767946.931181:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225167:225167:256:4294967295] 192.168.202.156@tcp LPNI seq info [225163:225163:8:4294967295] 00000400:00000200:0.0:1775767946.931184:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767946.931188:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775767946.931191:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767946.931194:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775767946.931197:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775767946.931199:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775767946.931202:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767946.931203:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1775767946.931206:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767946.931207:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775767946.931210:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775767946.931212:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775767946.931214:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775767946.931216:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775767946.931218:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775767946.931220:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfc700 x1862022294255232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767971 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767946.931226:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767946.931228:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfc700 x1862022294255232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767971 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767946.931232:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767946.931233:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775767946.931235:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767946.931237:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:0.0:1775767946.931239:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775767946.931241:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299871782 00000100:00000001:0.0:1775767946.931243:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299873085 00000100:00000001:0.0:1775767946.931245:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299871782 00000100:00000001:0.0:1775767946.931246:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299873085 00000100:00000001:0.0:1775767946.931247:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775767946.931248:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767946.931249:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775767946.931251:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767946/real 1775767946] req@ffff994207dfc700 x1862022294255232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775767946.931256:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775767946.931257:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775767946.931259:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775767946.931261:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775767946.931263:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775767946.931265:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775767946.931268:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfc700 x1862022294255232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775767946.931276:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfc700 x1862022294255232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767946.931289:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775767946.931291:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767946.931293:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767946.931294:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775767946.931298:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfc700 x1862022294255232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767946.931306:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775767946.931310:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfc700 x1862022294255232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767971 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767946.931319:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775767946.931322:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767946.931323:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767946.931325:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767946.931327:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775767946.931332:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775767946.931334:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767946.931338:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775767946.931340:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775767946.931344:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfc700 x1862022294255232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767971 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767946.931353:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294255232:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775767946.931357:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767946.931359:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767946.931363:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfce00 x1862022294254848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767971 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767946.931370:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775767946.931372:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775767946.931375:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002b800. 02000000:00000001:0.0:1775767946.931378:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775767946.931379:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775767946.931380:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:0.0:1775767946.931383:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775767946.931384:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1f800. 02000000:00000001:0.0:1775767946.931386:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775767946.931387:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775767946.931388:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfce00. 00000100:00000001:0.0:1775767946.931390:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775767946.931391:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767946.931393:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767946.931395:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfed80 x1862022294254976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767971 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767946.931400:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775767946.931401:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775767946.931402:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220029400. 02000000:00000001:0.0:1775767946.931403:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775767946.931404:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775767946.931405:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:0.0:1775767946.931406:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775767946.931418:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1d000. 02000000:00000001:0.0:1775767946.931421:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775767946.931421:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775767946.931422:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfed80. 00000100:00000001:0.0:1775767946.931424:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775767946.931425:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767946.931426:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767946.931429:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfe300 x1862022294255104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767971 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767946.931437:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775767946.931438:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775767946.931439:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002bc00. 02000000:00000001:0.0:1775767946.931441:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775767946.931442:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775767946.931444:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:0.0:1775767946.931446:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775767946.931448:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1f400. 02000000:00000001:0.0:1775767946.931450:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775767946.931451:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775767946.931452:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfe300. 00000100:00000001:0.0:1775767946.931455:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775767946.931456:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767946.931458:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767946.931461:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfc700 x1862022294255232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767971 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767946.931469:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775767946.931470:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775767946.931472:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220028400. 02000000:00000001:0.0:1775767946.931474:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775767946.931475:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775767946.931477:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:0.0:1775767946.931479:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775767946.931480:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1fc00. 02000000:00000001:0.0:1775767946.931482:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775767946.931483:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775767946.931485:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfc700. 00000100:00000001:0.0:1775767946.931487:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775767946.931488:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767946.931489:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767946.931492:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767946.931494:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767946.931495:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775767946.931496:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767946.993347:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767946.993355:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767946.993362:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767946.993364:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767946.993367:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767946.993368:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767946.993371:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767946.993372:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767946.993375:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767946.993377:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767946.993436:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767946.993440:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767946.993444:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767947.057231:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767947.057238:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767947.057240:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767947.057246:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767947.057250:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767947.057254:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767947.057256:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767947.057258:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767947.057261:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767947.953278:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767947.953278:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767947.953287:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767947.953288:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767947.953291:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767947.953292:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767947.953296:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:0.0:1775767947.953299:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775767947.953299:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767947.953306:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767947.953307:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767947.953307:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767947.953310:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767947.953313:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767947.953313:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767947.953314:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767947.953318:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767947.953325:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767947.953326:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00100000:1.0:1775767947.953327:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:0.0:1775767947.953330:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767947.953338:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767947.953338:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767947.953347:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767947.953353:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767947.953356:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767947.953359:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767948.017232:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767948.017237:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767948.017242:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767948.017243:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767948.017245:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767948.017246:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767948.017248:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767948.017249:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767948.017251:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767948.017252:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767948.017266:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767948.017268:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767948.017270:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767948.081231:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767948.081236:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767948.081238:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767948.081244:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767948.081248:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767948.081252:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767948.081254:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767948.081257:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767948.081259:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767948.273234:0:14953:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4299873085, at 4299873088 00000100:00000040:1.0:1775767948.273234:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299873085, at 4299873088 00000100:00000010:1.0:1775767948.273252:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3a80. 00000100:00000010:0.0:1775767948.273258:0:14953:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3d80. 00000100:00000001:1.0:1775767948.977212:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767948.977217:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767948.977221:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767948.977222:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767948.977224:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767948.977225:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767948.977232:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:0.0:1775767948.977236:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767948.977237:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767948.977242:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767948.977242:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767948.977244:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767948.977246:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767948.977250:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767948.977254:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767948.977254:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767948.977257:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767948.977261:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767948.977288:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767948.977292:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767948.977294:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767948.977297:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775767948.977300:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767948.977301:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767948.977303:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767948.977304:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767948.977306:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767949.041253:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767949.041260:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767949.041266:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767949.041267:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767949.041272:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767949.041273:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767949.041277:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767949.041279:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767949.041283:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767949.041284:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767949.041308:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767949.041312:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767949.041316:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767949.105294:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767949.105301:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767949.105303:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767949.105308:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767949.105312:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767949.105316:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767949.105318:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767949.105322:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767949.105325:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767950.001234:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767950.001241:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767950.001247:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767950.001248:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767950.001248:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767950.001251:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767950.001252:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767950.001254:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767950.001257:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767950.001262:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:1.0:1775767950.001262:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775767950.001270:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775767950.001271:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767950.001271:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767950.001277:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767950.001279:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767950.001279:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767950.001281:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767950.001287:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767950.001291:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767950.001295:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767950.001295:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767950.001299:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767950.001302:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767950.001302:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767950.001305:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767950.001308:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767950.065258:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767950.065264:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767950.065269:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767950.065270:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767950.065273:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767950.065274:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767950.065277:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767950.065278:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767950.065280:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767950.065281:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767950.065301:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767950.065305:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767950.065308:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767950.129225:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767950.129233:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767950.129235:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767950.129241:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767950.129245:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767950.129249:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767950.129252:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767950.129254:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767950.129257:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767951.025205:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767951.025213:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767951.025217:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767951.025223:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767951.025226:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767951.025229:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767951.025235:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767951.025239:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767951.025242:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767951.025271:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767951.025276:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767951.025278:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767951.025281:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767951.025281:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767951.025292:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767951.025293:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767951.025301:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767951.025302:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767951.025307:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767951.025311:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767951.025312:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:0.0:1775767951.025313:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767951.025323:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767951.025328:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767951.025332:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767951.025335:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767951.025339:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767951.089259:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767951.089267:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775767951.089270:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775767951.089273:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767951.089278:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775767951.089280:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767951.089283:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767951.089285:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767951.089290:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767951.089291:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767951.089295:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767951.089297:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767951.089300:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767951.089302:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767951.089306:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767951.089307:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767951.089329:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767951.089332:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767951.089335:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767951.153257:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767951.153263:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767951.153265:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767951.153270:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767951.153273:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767951.153276:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767951.153278:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767951.153279:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767951.153281:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767952.049218:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767952.049225:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767952.049226:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767952.049227:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767952.049232:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767952.049235:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767952.049240:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:0.0:1775767952.049242:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767952.049250:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767952.049252:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767952.049256:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767952.049256:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767952.049264:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767952.049267:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767952.049271:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767952.049271:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767952.049277:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767952.049282:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767952.049287:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767952.049296:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:2.0:1775767952.049296:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775767952.049301:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775767952.049305:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:1.0:1775767952.049307:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775767952.049309:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775767952.049314:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1775767952.049314:0:8:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775767952.049320:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775767952.049321:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767952.049326:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:2.0:1775767952.049328:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5204 00000100:00000001:1.0:1775767952.049331:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767952.049336:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767952.049342:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767952.049351:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775767952.049354:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775767952.049370:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767952.049374:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767952.049378:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767952.049380:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775767952.049385:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775767952.049387:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:2.0:1775767952.049389:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775767952.049392:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:2.0:1775767952.049394:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:2.0:1775767952.049396:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:2.0:1775767952.049398:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767952.049400:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767952.049401:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767952.049403:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775767952.049405:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:2.0:1775767952.049408:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775767952.049410:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767952.049413:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775767952.049416:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767952.049418:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767952.049420:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775767952.049422:0:8:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775767952.049432:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dff100. 00000020:00000040:2.0:1775767952.049435:0:8:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:2.0:1775767952.049438:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775767952.049440:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767952.049442:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775767952.049446:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775767952.049451:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775767952.049459:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1d400. 00000100:00000001:2.0:1775767952.049466:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767952.049482:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dff100 x1862022294255360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775767952.049492:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775767952.049496:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767952.049500:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3e64ae1] to pc [ptlrpcd_rcv+-1] req@ffff994207dff100 x1862022294255360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767952.049519:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775767952.049522:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767952.049524:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775767952.049525:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775767952.049527:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775767952.049529:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775767952.049531:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5204 00000400:00000200:2.0:1775767952.049534:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767952.049537:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775767952.049539:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775767952.049541:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775767952.049544:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775767952.049544:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1775767952.049552:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:2.0:1775767952.049552:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767952.049558:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:0.0:1775767952.049559:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000001:2.0:1775767952.049562:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775767952.049563:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:2.0:1775767952.049564:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775767952.049566:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:2.0:1775767952.049572:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:0.0:1775767952.049573:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dff100 x1862022294255360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1775767952.049576:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:2.0:1775767952.049588:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767952.049604:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1775767952.049605:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767952.049610:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767952.049616:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000001:0.0:1775767952.049617:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1775767952.049623:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 02000000:00000001:0.0:1775767952.049624:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775767952.049630:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775767952.049633:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00100000:0.0:1775767952.049636:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294255360:192.168.202.156@tcp:250:kworker.0 00000100:00000001:0.0:1775767952.049645:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00080000:2.0:1775767952.049645:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 02000000:00000001:0.0:1775767952.049654:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1775767952.049654:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767952.049658:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767952.049661:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775767952.049662:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775767952.049673:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220029400. 02000000:00000001:2.0:1775767952.049674:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775767952.049679:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767952.049681:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767952.049686:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775767952.049691:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000400:00000200:0.0:1775767952.049699:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00000001:2.0:1775767952.049700:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767952.049704:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775767952.049705:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000100:00000010:2.0:1775767952.049709:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfdc00. 00000400:00000010:0.0:1775767952.049713:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000020:00000040:2.0:1775767952.049713:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:2.0:1775767952.049717:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775767952.049719:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767952.049721:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775767952.049723:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000200:0.0:1775767952.049724:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294255360, portal 25 02000000:00000001:2.0:1775767952.049726:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767952.049727:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775767952.049732:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611848960 : -112966097702656 : ffff994207dff100) 02000000:00000010:2.0:1775767952.049733:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1c800. 00000100:00000001:2.0:1775767952.049740:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767952.049742:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dff100 x1862022294255360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767977 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767952.049754:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfdc00 x1862022294255488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775767952.049755:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775767952.049764:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775767952.049769:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775767952.049780:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294255360, offset 0 00000100:00000001:2.0:1775767952.049792:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775767952.049796:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767952.049799:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775767952.049811:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000040:2.0:1775767952.049818:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003e25603c] to pc [ptlrpcd_rcv+-1] req@ffff994207dfdc00 x1862022294255488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775767952.049822:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775767952.049833:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775767952.049843:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000100:00000001:2.0:1775767952.049843:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775767952.049851:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1775767952.049855:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000100:00080000:2.0:1775767952.049864:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000200:0.0:1775767952.049865:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00000001:2.0:1775767952.049870:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775767952.049872:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:0.0:1775767952.049880:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225168:225168:256:4294967295] 192.168.202.156@tcp LPNI seq info [225164:225164:8:4294967295] 00000100:00000001:2.0:1775767952.049880:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775767952.049886:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5204 00000400:00000200:0.0:1775767952.049901:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767952.049909:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767952.049922:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775767952.049925:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775767952.049937:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000200:0.0:1775767952.049941:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000100:00000001:2.0:1775767952.049946:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767952.049954:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000800:00000200:0.0:1775767952.049956:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00000001:2.0:1775767952.049960:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000800:00000010:0.0:1775767952.049965:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000020:00000001:2.0:1775767952.049966:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775767952.049968:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:2.0:1775767952.049980:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000800:00000200:0.0:1775767952.049982:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000040:2.0:1775767952.049985:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000800:00000200:0.0:1775767952.049988:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000020:00000001:2.0:1775767952.049990:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:2.0:1775767952.050002:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000800:00000200:0.1:1775767952.050006:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767952.050012:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000100:00000040:2.0:1775767952.050013:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767952.050022:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000100:0.0:1775767952.050027:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000100:00000040:2.0:1775767952.050029:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000800:00000010:0.0:1775767952.050032:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000100:00000001:2.0:1775767952.050034:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775767952.050040:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000400:00000200:0.0:1775767952.050042:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00080000:2.0:1775767952.050046:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775767952.050050:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767952.050056:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:0.0:1775767952.050057:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 02000000:00000001:2.0:1775767952.050065:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1775767952.050067:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 02000000:00000001:2.0:1775767952.050071:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775767952.050072:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:2.0:1775767952.050078:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775767952.050081:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000008:00100000:2.0:1775767952.050083:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00000200:0.0:1775767952.050091:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dff100 x1862022294255360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767977 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:2.0:1775767952.050091:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767952.050098:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775767952.050108:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfc700. 00000020:00000040:2.0:1775767952.050113:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000100:00000001:0.0:1775767952.050114:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1775767952.050120:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775767952.050122:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767952.050124:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dff100 x1862022294255360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767977 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767952.050126:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775767952.050128:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775767952.050131:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775767952.050138:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1ec00. 00000100:00000001:2.0:1775767952.050148:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767952.050149:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767952.050152:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:2.0:1775767952.050193:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfc700 x1862022294255616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775767952.050198:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767952.050210:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:0.0:1775767952.050211:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767952.050216:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767952.050218:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299878206 00000100:00000040:2.0:1775767952.050224:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a176dd1] to pc [ptlrpcd_rcv+-1] req@ffff994207dfc700 x1862022294255616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1775767952.050227:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942078a3d80. 00000100:00000001:0.0:1775767952.050238:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775767952.050247:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299878206 00000100:00000001:2.0:1775767952.050247:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775767952.050252:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000010:0.0:1775767952.050254:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942078a3ae0. 00000100:00000001:0.0:1775767952.050261:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00080000:2.0:1775767952.050262:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1775767952.050267:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1775767952.050269:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1775767952.050270:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775767952.050278:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767952.050280:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:0.0:1775767952.050285:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00080000:2.0:1775767952.050288:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5204 00000100:00100000:0.0:1775767952.050296:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767952/real 1775767952] req@ffff994207dff100 x1862022294255360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1775767952.050297:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767952.050311:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775767952.050320:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000400:00000001:0.0:1775767952.050329:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:2.0:1775767952.050330:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000001:0.0:1775767952.050334:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1775767952.050334:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767952.050339:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000010:0.0:1775767952.050340:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000100:00000001:2.0:1775767952.050344:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775767952.050348:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775767952.050349:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000400:00000200:0.0:1775767952.050350:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000020:00000001:2.0:1775767952.050351:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775767952.050356:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000400:00000010:0.0:1775767952.050372:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000020:00000001:2.0:1775767952.050373:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:2.0:1775767952.050379:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:2.0:1775767952.050383:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775767952.050385:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1775767952.050385:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767952.050389:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767952.050392:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775767952.050393:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00000200:0.0:1775767952.050398:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dff100 x1862022294255360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767952.050399:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775767952.050403:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767952.050424:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775767952.050430:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767952.050434:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1775767952.050435:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dff100 x1862022294255360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767952.050437:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775767952.050441:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:2.0:1775767952.050451:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767952.050457:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775767952.050468:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfe300. 00000100:00000001:0.0:1775767952.050472:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000040:2.0:1775767952.050472:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:2.0:1775767952.050479:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1775767952.050480:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1775767952.050481:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767952.050487:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775767952.050489:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1775767952.050490:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:2.0:1775767952.050492:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767952.050494:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:2.0:1775767952.050496:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1f000. 00000100:00000001:2.0:1775767952.050501:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767952.050506:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dff100 x1862022294255360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767952.050508:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfe300 x1862022294255744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775767952.050540:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767952.050542:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775767952.050544:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767952.050554:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c661fef] to pc [ptlrpcd_rcv+-1] req@ffff994207dfe300 x1862022294255744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775767952.050558:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dff100 x1862022294255360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767977 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767952.050575:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775767952.050580:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767952.050594:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1775767952.050599:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775767952.050601:0:8:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775767952.050609:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:0.0:1775767952.050612:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767952.050613:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5204 00000100:00000001:0.0:1775767952.050624:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1775767952.050625:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767952.050630:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:0.0:1775767952.050635:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:2.0:1775767952.050635:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1775767952.050642:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767952.050642:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767952.050648:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775767952.050652:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775767952.050653:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767952.050655:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775767952.050659:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:0.0:1775767952.050660:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000040:2.0:1775767952.050662:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:2.0:1775767952.050665:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775767952.050668:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000100:00080000:0.0:1775767952.050672:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000020:00000001:2.0:1775767952.050672:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:2.0:1775767952.050679:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:2.0:1775767952.050686:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767952.050689:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00080000:0.0:1775767952.050690:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000040:2.0:1775767952.050695:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767952.050701:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:0.0:1775767952.050703:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000040:2.0:1775767952.050704:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:2.0:1775767952.050713:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000040:0.0:1775767952.050728:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dff100 x1862022294255360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767977 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767952.050732:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767952.050740:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775767952.050745:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767952.050754:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767952.050761:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294255360:192.168.202.156@tcp:250:kworker.0 00000100:00000001:2.0:1775767952.050761:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775767952.050765:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000100:00000001:0.0:1775767952.050771:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1775767952.050772:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767952.050778:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1775767952.050779:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767952.050786:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dff100 x1862022294255360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767977 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1775767952.050786:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfed80. 00000020:00000040:2.0:1775767952.050793:0:8:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:2.0:1775767952.050804:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775767952.050807:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767952.050817:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1775767952.050818:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767952.050822:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775767952.050824:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1775767952.050826:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775767952.050828:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220029400. 02000000:00000001:0.0:1775767952.050835:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000010:2.0:1775767952.050835:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1e800. 00000020:00000001:0.0:1775767952.050839:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:2.0:1775767952.050842:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1775767952.050843:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:0.0:1775767952.050849:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775767952.050853:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1d400. 00000100:00100000:2.0:1775767952.050855:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfed80 x1862022294255872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1775767952.050861:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775767952.050866:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775767952.050869:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dff100. 00000100:00000001:0.0:1775767952.050885:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775767952.050886:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767952.050892:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775767952.050893:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767952.050897:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767952.050905:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:2.0:1775767952.050905:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb9b9eae] to pc [ptlrpcd_rcv+-1] req@ffff994207dfed80 x1862022294255872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767952.050909:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767952.050917:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775767952.050923:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767952.050926:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1775767952.050926:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767952.050935:0:8:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5214) 00000100:00000040:0.0:1775767952.050936:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfdc00 x1862022294255488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767952.050946:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767952.050948:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767952.050951:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775767952.050953:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775767952.050956:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767952.050963:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294255488:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775767952.050967:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775767952.050969:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775767952.050971:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767952.050973:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775767952.050976:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220029400. 02000000:00000001:0.0:1775767952.050978:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767952.050982:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775767952.050985:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775767952.050988:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775767952.050991:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294255488, portal 10 00000100:00000001:0.0:1775767952.051004:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775767952.051005:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611843584 : -112966097708032 : ffff994207dfdc00) 00000100:00000040:0.0:1775767952.051011:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfdc00 x1862022294255488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767977 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767952.051017:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775767952.051020:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775767952.051022:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775767952.051024:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294255488, offset 0 00000400:00000200:0.0:1775767952.051027:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775767952.051032:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767952.051036:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775767952.051039:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775767952.051042:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775767952.051044:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775767952.051046:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775767952.051049:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225169:225169:256:4294967295] 192.168.202.156@tcp LPNI seq info [225165:225165:8:4294967295] 00000400:00000200:0.0:1775767952.051053:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767952.051071:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775767952.051075:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767952.051078:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775767952.051082:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775767952.051085:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775767952.051089:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767952.051092:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1775767952.051096:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767952.051098:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775767952.051101:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775767952.051115:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775767952.051120:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775767952.051122:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775767952.051125:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775767952.051128:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfdc00 x1862022294255488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767977 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767952.051136:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767952.051138:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfdc00 x1862022294255488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767977 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767952.051144:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767952.051146:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775767952.051148:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767952.051152:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:0.0:1775767952.051185:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775767952.051188:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299876884 00000100:00000001:0.0:1775767952.051192:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299878206 00000100:00000001:0.0:1775767952.051195:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299876884 00000100:00000001:0.0:1775767952.051198:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299878206 00000100:00000001:0.0:1775767952.051201:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775767952.051204:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767952.051206:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775767952.051212:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767952/real 1775767952] req@ffff994207dfdc00 x1862022294255488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775767952.051221:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775767952.051223:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775767952.051225:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775767952.051228:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775767952.051231:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775767952.051233:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775767952.051236:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfdc00 x1862022294255488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775767952.051245:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfdc00 x1862022294255488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767952.051251:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775767952.051253:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767952.051255:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767952.051256:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775767952.051260:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfdc00 x1862022294255488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767952.051267:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775767952.051271:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfdc00 x1862022294255488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767977 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767952.051278:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775767952.051281:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767952.051283:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767952.051284:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767952.051286:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775767952.051289:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775767952.051291:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767952.051295:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775767952.051297:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775767952.051301:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfdc00 x1862022294255488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767977 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767952.051309:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294255488:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775767952.051314:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775767952.051317:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfc700 x1862022294255616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767952.051324:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767952.051326:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767952.051328:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775767952.051330:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775767952.051332:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767952.051335:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294255616:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775767952.051338:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775767952.051340:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775767952.051342:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767952.051344:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775767952.051347:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002b800. 02000000:00000001:0.0:1775767952.051349:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767952.051352:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775767952.051355:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775767952.051375:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775767952.051379:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294255616, portal 10 00000100:00000001:0.0:1775767952.051381:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775767952.051382:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611838208 : -112966097713408 : ffff994207dfc700) 00000100:00000040:0.0:1775767952.051389:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfc700 x1862022294255616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767977 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767952.051401:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775767952.051403:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775767952.051405:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775767952.051408:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294255616, offset 0 00000400:00000200:0.0:1775767952.051411:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775767952.051416:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767952.051420:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775767952.051423:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775767952.051425:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775767952.051427:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775767952.051430:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775767952.051433:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225170:225170:256:4294967295] 192.168.202.156@tcp LPNI seq info [225166:225166:8:4294967295] 00000400:00000200:0.0:1775767952.051437:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767952.051444:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775767952.051449:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767952.051452:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775767952.051456:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775767952.051459:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775767952.051463:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767952.051466:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1775767952.051470:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767952.051472:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775767952.051475:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775767952.051478:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775767952.051481:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775767952.051483:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775767952.051485:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775767952.051488:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfc700 x1862022294255616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767977 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767952.051496:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767952.051499:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfc700 x1862022294255616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767977 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767952.051511:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767952.051513:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775767952.051516:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767952.051519:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:0.0:1775767952.051522:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775767952.051524:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299876876 00000100:00000001:0.0:1775767952.051527:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299878206 00000100:00000001:0.0:1775767952.051529:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299876876 00000100:00000001:0.0:1775767952.051531:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299878206 00000100:00000001:0.0:1775767952.051533:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775767952.051535:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767952.051537:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775767952.051541:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767952/real 1775767952] req@ffff994207dfc700 x1862022294255616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775767952.051548:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775767952.051549:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775767952.051551:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775767952.051553:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775767952.051555:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775767952.051557:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775767952.051560:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfc700 x1862022294255616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775767952.051568:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfc700 x1862022294255616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767952.051574:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775767952.051576:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767952.051577:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767952.051579:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775767952.051582:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfc700 x1862022294255616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767952.051590:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775767952.051593:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfc700 x1862022294255616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767977 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767952.051600:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775767952.051603:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767952.051605:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767952.051606:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767952.051608:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775767952.051611:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775767952.051614:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767952.051618:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775767952.051621:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775767952.051625:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfc700 x1862022294255616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767977 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767952.051633:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294255616:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775767952.051637:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775767952.051641:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfe300 x1862022294255744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767952.051649:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767952.051650:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767952.051653:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775767952.051655:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775767952.051657:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767952.051660:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294255744:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775767952.051663:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775767952.051665:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775767952.051667:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767952.051669:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775767952.051672:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002bc00. 02000000:00000001:0.0:1775767952.051674:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767952.051678:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775767952.051681:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775767952.051685:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775767952.051687:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294255744, portal 4 00000100:00000001:0.0:1775767952.051689:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775767952.051691:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611845376 : -112966097706240 : ffff994207dfe300) 00000100:00000040:0.0:1775767952.051696:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfe300 x1862022294255744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767977 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767952.051703:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775767952.051705:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775767952.051708:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775767952.051710:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294255744, offset 0 00000400:00000200:0.0:1775767952.051714:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775767952.051718:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767952.051721:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775767952.051725:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775767952.051727:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775767952.051730:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775767952.051732:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775767952.051735:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225171:225171:256:4294967295] 192.168.202.156@tcp LPNI seq info [225167:225167:8:4294967295] 00000400:00000200:0.0:1775767952.051740:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767952.051749:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775767952.051753:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767952.051757:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775767952.051761:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775767952.051764:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775767952.051769:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767952.051772:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1775767952.051776:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767952.051778:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775767952.051781:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775767952.051785:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775767952.051788:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775767952.051790:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775767952.051792:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775767952.051796:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfe300 x1862022294255744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767977 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767952.051804:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767952.051806:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfe300 x1862022294255744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767977 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767952.051813:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767952.051814:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775767952.051817:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767952.051820:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:0.0:1775767952.051823:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775767952.051825:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299876899 00000100:00000001:0.0:1775767952.051828:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299878206 00000100:00000001:0.0:1775767952.051830:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299876899 00000100:00000001:0.0:1775767952.051832:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299878206 00000100:00000001:0.0:1775767952.051834:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775767952.051836:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767952.051838:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775767952.051842:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767952/real 1775767952] req@ffff994207dfe300 x1862022294255744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775767952.051850:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775767952.051851:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775767952.051853:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775767952.051855:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775767952.051857:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775767952.051860:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775767952.051863:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfe300 x1862022294255744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775767952.051872:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfe300 x1862022294255744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767952.051878:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775767952.051880:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767952.051882:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767952.051883:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775767952.051887:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfe300 x1862022294255744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767952.051893:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775767952.051897:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfe300 x1862022294255744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767977 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767952.051904:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775767952.051906:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767952.051908:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767952.051909:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767952.051911:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775767952.051914:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775767952.051917:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767952.051920:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775767952.051922:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775767952.051926:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfe300 x1862022294255744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767977 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767952.051934:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294255744:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775767952.051938:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767952.051939:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767952.051943:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfdc00 x1862022294255488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767977 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767952.051950:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775767952.051951:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775767952.051953:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220029400. 02000000:00000001:0.0:1775767952.051956:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775767952.051957:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775767952.051958:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:0.0:1775767952.051961:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775767952.051963:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1c800. 02000000:00000001:0.0:1775767952.051965:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775767952.051967:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775767952.051968:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfdc00. 00000100:00000001:0.0:1775767952.051971:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775767952.051972:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767952.051974:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767952.051977:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfc700 x1862022294255616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767977 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767952.051984:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775767952.051985:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775767952.051986:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002b800. 02000000:00000001:0.0:1775767952.051989:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775767952.051990:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775767952.051991:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:0.0:1775767952.051993:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775767952.051994:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1ec00. 02000000:00000001:0.0:1775767952.051997:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775767952.051998:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775767952.052000:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfc700. 00000100:00000001:0.0:1775767952.052002:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775767952.052003:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767952.052005:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767952.052008:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfe300 x1862022294255744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767977 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767952.052014:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775767952.052016:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775767952.052017:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002bc00. 02000000:00000001:0.0:1775767952.052019:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775767952.052021:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775767952.052022:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:0.0:1775767952.052024:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775767952.052025:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1f000. 02000000:00000001:0.0:1775767952.052027:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775767952.052028:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775767952.052030:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfe300. 00000100:00000001:0.0:1775767952.052032:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775767952.052033:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767952.052035:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767952.052038:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767952.052040:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767952.052041:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775767952.052042:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767952.052043:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775767952.052047:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfed80 x1862022294255872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767952.052052:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767952.052053:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767952.052055:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775767952.052056:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775767952.052058:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767952.052059:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294255872:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775767952.052061:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775767952.052063:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775767952.052064:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767952.052066:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775767952.052068:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002bc00. 02000000:00000001:0.0:1775767952.052069:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767952.052072:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775767952.052073:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775767952.052076:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775767952.052078:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294255872, portal 4 00000100:00000001:0.0:1775767952.052080:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775767952.052081:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611848064 : -112966097703552 : ffff994207dfed80) 00000100:00000040:0.0:1775767952.052085:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfed80 x1862022294255872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767977 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767952.052089:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775767952.052091:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775767952.052092:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775767952.052094:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294255872, offset 0 00000400:00000200:0.0:1775767952.052096:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775767952.052108:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767952.052110:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775767952.052112:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775767952.052114:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775767952.052116:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775767952.052117:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775767952.052119:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225172:225172:256:4294967295] 192.168.202.156@tcp LPNI seq info [225168:225168:8:4294967295] 00000400:00000200:0.0:1775767952.052122:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767952.052126:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775767952.052129:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767952.052132:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775767952.052134:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775767952.052136:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775767952.052139:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767952.052141:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1775767952.052143:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767952.052145:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775767952.052147:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775767952.052149:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775767952.052151:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775767952.052153:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775767952.052171:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775767952.052174:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfed80 x1862022294255872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767977 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767952.052184:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767952.052187:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfed80 x1862022294255872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767977 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767952.052191:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767952.052193:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775767952.052195:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767952.052197:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:0.0:1775767952.052199:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775767952.052200:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299876903 00000100:00000001:0.0:1775767952.052202:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299878206 00000100:00000001:0.0:1775767952.052204:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299876903 00000100:00000001:0.0:1775767952.052205:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299878206 00000100:00000001:0.0:1775767952.052206:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775767952.052208:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767952.052209:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775767952.052211:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767952/real 1775767952] req@ffff994207dfed80 x1862022294255872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775767952.052216:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775767952.052217:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775767952.052218:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775767952.052219:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775767952.052220:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775767952.052222:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775767952.052223:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfed80 x1862022294255872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775767952.052228:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfed80 x1862022294255872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767952.052233:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775767952.052234:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767952.052235:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767952.052236:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775767952.052238:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfed80 x1862022294255872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767952.052242:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775767952.052245:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfed80 x1862022294255872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767977 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767952.052248:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775767952.052250:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767952.052251:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767952.052252:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767952.052253:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775767952.052255:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775767952.052257:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767952.052259:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775767952.052260:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775767952.052271:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfed80 x1862022294255872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767977 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767952.052276:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294255872:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775767952.052278:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767952.052279:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767952.052281:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfed80 x1862022294255872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767977 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767952.052285:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775767952.052286:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775767952.052287:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002bc00. 02000000:00000001:0.0:1775767952.052289:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775767952.052289:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775767952.052290:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:0.0:1775767952.052292:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775767952.052293:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1e800. 02000000:00000001:0.0:1775767952.052295:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775767952.052295:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775767952.052296:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfed80. 00000100:00000001:0.0:1775767952.052298:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775767952.052299:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767952.052300:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767952.052302:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767952.052303:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767952.052304:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775767952.052305:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767952.113228:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767952.113234:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767952.113241:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767952.113243:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767952.113248:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767952.113249:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767952.113253:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767952.113255:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767952.113259:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767952.113261:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767952.113284:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767952.113287:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767952.113289:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767952.177217:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767952.177224:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767952.177225:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767952.177231:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767952.177234:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767952.177239:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767952.177242:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767952.177244:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767952.177248:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767953.073223:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767953.073224:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767953.073229:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767953.073230:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767953.073232:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767953.073234:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767953.073238:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775767953.073239:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775767953.073243:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767953.073245:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767953.073246:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767953.073249:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767953.073250:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767953.073251:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767953.073251:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767953.073258:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767953.073258:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767953.073258:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767953.073263:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767953.073264:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767953.073268:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767953.073270:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767953.073275:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767953.073280:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767953.073283:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767953.073285:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767953.073287:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767953.137259:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767953.137265:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767953.137272:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767953.137274:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767953.137277:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767953.137278:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767953.137282:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767953.137283:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767953.137286:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767953.137288:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767953.137310:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767953.137313:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767953.137316:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767953.201281:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767953.201287:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767953.201288:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767953.201293:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767953.201296:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767953.201300:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767953.201303:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767953.201305:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767953.201308:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767953.393264:0:14953:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4299878206, at 4299878208 00000100:00000040:1.0:1775767953.393264:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299878206, at 4299878208 00000100:00000010:0.0:1775767953.393279:0:14953:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3d80. 00000100:00000010:1.0:1775767953.393281:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3ae0. 00000100:00000001:3.0:1775767954.097220:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767954.097226:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767954.097229:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767954.097235:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767954.097240:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767954.097243:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767954.097244:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767954.097244:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767954.097248:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767954.097253:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767954.097253:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767954.097253:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767954.097257:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767954.097258:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767954.097260:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767954.097269:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:0.0:1775767954.097270:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767954.097271:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767954.097272:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767954.097280:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767954.097282:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767954.097284:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767954.097286:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767954.097291:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767954.097291:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767954.097296:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767954.097299:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767954.161280:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767954.161286:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767954.161291:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767954.161292:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767954.161295:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767954.161296:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767954.161299:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767954.161300:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767954.161302:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767954.161303:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767954.161332:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767954.161335:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767954.161363:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767954.225268:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767954.225276:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767954.225278:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767954.225284:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767954.225287:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767954.225291:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767954.225294:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767954.225296:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767954.225299:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767955.121283:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767955.121283:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767955.121283:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767955.121291:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767955.121292:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767955.121292:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767955.121295:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767955.121296:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767955.121304:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:0.0:1775767955.121305:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767955.121305:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:0.0:1775767955.121312:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767955.121313:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767955.121314:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767955.121318:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767955.121320:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767955.121320:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767955.121321:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767955.121325:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767955.121345:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767955.121364:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767955.121366:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767955.121369:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767955.121373:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767955.121374:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767955.121377:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767955.121380:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767955.185242:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767955.185248:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767955.185253:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767955.185254:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767955.185257:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767955.185259:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767955.185262:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767955.185263:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767955.185266:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767955.185267:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767955.185286:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767955.185288:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767955.185291:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767955.249289:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767955.249296:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767955.249297:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767955.249303:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767955.249307:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767955.249311:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767955.249314:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767955.249316:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767955.249318:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767956.145252:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767956.145252:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767956.145259:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767956.145259:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767956.145261:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767956.145263:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767956.145268:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775767956.145268:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775767956.145274:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767956.145274:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767956.145279:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767956.145279:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767956.145285:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767956.145287:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767956.145289:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767956.145294:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767956.145296:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767956.145299:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767956.145374:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767956.145379:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767956.145381:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767956.145387:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775767956.145391:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767956.145394:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767956.145397:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767956.145401:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767956.145405:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767956.209268:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767956.209274:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767956.209281:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767956.209286:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775767956.209288:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775767956.209291:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767956.209294:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775767956.209297:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767956.209299:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767956.209302:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767956.209306:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767956.209307:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775767956.209309:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775767956.209311:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767956.209313:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775767956.209314:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767956.209316:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767956.209332:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767956.209338:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767956.209340:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775767956.209341:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775767956.209343:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767956.209344:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775767956.209346:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767956.209347:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767956.209350:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767956.209353:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767956.209354:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775767956.209356:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775767956.209358:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767956.209360:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775767956.209361:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767956.209363:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767956.209365:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767956.209400:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767956.209404:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767956.209407:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767956.273247:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767956.273253:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767956.273255:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767956.273260:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767956.273264:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767956.273268:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767956.273270:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767956.273273:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767956.273275:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767957.169289:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775767957.169292:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767957.169294:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:3.0:1775767957.169299:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775767957.169302:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767957.169303:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767957.169304:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1775767957.169321:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767957.169321:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767957.169325:0:8:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775767957.169330:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:2.0:1775767957.169331:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:0.0:1775767957.169332:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767957.169332:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775767957.169336:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767957.169344:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767957.169345:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767957.169345:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767957.169346:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5209 00000100:00100000:1.0:1775767957.169354:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:0.0:1775767957.169355:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767957.169356:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767957.169363:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000400:00000200:3.0:1775767957.169369:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775767957.169370:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767957.169371:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767957.169378:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1775767957.169384:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:0.0:1775767957.169385:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767957.169386:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:3.0:1775767957.169387:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1775767957.169389:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767957.169400:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767957.169401:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767957.169401:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767957.169407:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767957.169412:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775767957.169414:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767957.169419:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767957.169421:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767957.169424:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767957.169426:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:3.0:1775767957.169430:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767957.169432:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:3.0:1775767957.169434:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:3.0:1775767957.169436:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:3.0:1775767957.169439:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767957.169441:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767957.169442:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767957.169444:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767957.169447:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:3.0:1775767957.169450:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767957.169452:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767957.169456:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767957.169459:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767957.169461:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767957.169464:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775767957.169466:0:8:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767957.169475:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aad80. 00000020:00000040:3.0:1775767957.169478:0:8:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:3.0:1775767957.169481:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767957.169482:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767957.169485:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767957.169487:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767957.169491:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767957.169499:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2000. 00000100:00000001:3.0:1775767957.169505:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767957.169512:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aad80 x1862022294256000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767957.169521:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767957.169525:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767957.169536:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000045ccf08] to pc [ptlrpcd_rcv+-1] req@ffff9942099aad80 x1862022294256000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767957.169552:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767957.169555:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767957.169558:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775767957.169559:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767957.169560:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767957.169562:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767957.169564:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5209 00000400:00000200:3.0:1775767957.169566:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767957.169569:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767957.169572:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767957.169575:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767957.169578:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767957.169582:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767957.169584:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767957.169588:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767957.169590:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:3.0:1775767957.169592:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767957.169595:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:3.0:1775767957.169597:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000001:0.0:1775767957.169599:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1775767957.169601:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000001:0.0:1775767957.169605:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1775767957.169609:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775767957.169612:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1775767957.169619:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767957.169632:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000040:0.0:1775767957.169633:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099aad80 x1862022294256000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767957.169642:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767957.169653:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00080000:3.0:1775767957.169665:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775767957.169677:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767957.169677:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767957.169682:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767957.169682:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767957.169689:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775767957.169691:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767957.169692:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767957.169695:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:0.0:1775767957.169696:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00100000:3.0:1775767957.169698:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1775767957.169703:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1775767957.169703:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767957.169713:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767957.169720:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099aad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294256000:192.168.202.156@tcp:250:kworker.0 00000100:00000010:3.0:1775767957.169721:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aa680. 00000020:00000040:3.0:1775767957.169727:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000100:00000001:0.0:1775767957.169735:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767957.169736:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767957.169740:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767957.169742:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767957.169743:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775767957.169745:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767957.169745:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767957.169751:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767957.169751:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0c00. 02000000:00000001:0.0:1775767957.169760:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1775767957.169762:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767957.169772:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aa680 x1862022294256128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000010:0.0:1775767957.169773:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002b800. 02000000:00000001:0.0:1775767957.169780:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767957.169795:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767957.169796:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767957.169803:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775767957.169804:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000100:00000040:3.0:1775767957.169810:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006cb1597d] to pc [ptlrpcd_rcv+-1] req@ffff9942099aa680 x1862022294256128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1775767957.169815:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000001:3.0:1775767957.169826:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767957.169828:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:0.0:1775767957.169830:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294256000, portal 25 00000100:00080000:3.0:1775767957.169834:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775767957.169837:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767957.169838:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1775767957.169840:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640864128 : -112966068687488 : ffff9942099aad80) 00000100:00080000:3.0:1775767957.169842:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767957.169848:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767957.169854:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5209 00000100:00000040:0.0:1775767957.169856:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099aad80 x1862022294256000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767982 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1775767957.169868:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767957.169880:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:0.0:1775767957.169884:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1775767957.169885:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775767957.169894:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000100:00000040:3.0:1775767957.169895:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000010:0.0:1775767957.169899:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000001:3.0:1775767957.169901:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:0.0:1775767957.169907:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294256000, offset 0 00000100:00000040:3.0:1775767957.169907:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767957.169916:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767957.169920:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767957.169922:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:3.0:1775767957.169927:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:0.0:1775767957.169932:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000020:00000040:3.0:1775767957.169933:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:3.0:1775767957.169937:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:3.0:1775767957.169939:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:3.0:1775767957.169942:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767957.169945:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1775767957.169951:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000040:3.0:1775767957.169951:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767957.169960:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767957.169963:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000400:00000200:0.0:1775767957.169970:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00080000:3.0:1775767957.169970:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767957.169978:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767957.169981:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:0.0:1775767957.169983:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:3.0:1775767957.169988:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1775767957.169994:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 02000000:00000001:3.0:1775767957.169994:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767957.169999:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767957.170002:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000400:00000200:0.0:1775767957.170003:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775767957.170011:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000008:00000001:3.0:1775767957.170011:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767957.170017:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767957.170019:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225173:225173:256:4294967295] 192.168.202.156@tcp LPNI seq info [225169:225169:8:4294967295] 00000400:00000200:0.0:1775767957.170027:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000010:3.0:1775767957.170035:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ab800. 00000020:00000040:3.0:1775767957.170043:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000400:00000200:0.0:1775767957.170049:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 02000000:00000001:3.0:1775767957.170051:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767957.170055:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767957.170061:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000800:00000200:0.0:1775767957.170062:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 02000000:00000001:3.0:1775767957.170065:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767957.170069:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1775767957.170071:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 02000000:00000010:3.0:1775767957.170076:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2800. 00000100:00000001:3.0:1775767957.170081:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1775767957.170082:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775767957.170091:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000100:00100000:3.0:1775767957.170093:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ab800 x1862022294256256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767957.170107:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767957.170113:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775767957.170116:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000040:3.0:1775767957.170119:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000089df80c0] to pc [ptlrpcd_rcv+-1] req@ffff9942099ab800 x1862022294256256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.1:1775767957.170125:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000100:00000001:3.0:1775767957.170141:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767957.170147:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1775767957.170150:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000100:00080000:3.0:1775767957.170195:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775767957.170198:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000800:00000010:0.0:1775767957.170199:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000100:00080000:3.0:1775767957.170201:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767957.170209:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767957.170214:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5209 00000400:00000200:0.0:1775767957.170216:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767957.170223:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767957.170230:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000400:00000200:0.0:1775767957.170233:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000100:00080000:3.0:1775767957.170235:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767957.170243:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000200:0.0:1775767957.170244:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775767957.170251:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:3.0:1775767957.170251:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767957.170258:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775767957.170259:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1775767957.170265:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000200:0.0:1775767957.170272:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099aad80 x1862022294256000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767982 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775767957.170272:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767957.170276:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:3.0:1775767957.170284:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767957.170287:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:3.0:1775767957.170297:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:3.0:1775767957.170306:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:0.0:1775767957.170325:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767957.170331:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000040:0.0:1775767957.170337:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099aad80 x1862022294256000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767982 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767957.170337:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767957.170346:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767957.170353:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767957.170360:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:3.0:1775767957.170369:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775767957.170370:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767957.170372:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767957.170377:0:2399:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:3.0:1775767957.170378:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767957.170385:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767957.170388:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767957.170390:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767957.170391:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767957.170395:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:3.0:1775767957.170399:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767957.170404:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767957.170405:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000010:3.0:1775767957.170408:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aa300. 00000100:00000001:0.0:1775767957.170411:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299883326 00000020:00000040:3.0:1775767957.170412:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:3.0:1775767957.170421:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767957.170423:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767957.170425:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000010:0.0:1775767957.170426:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942078a3d80. 02000000:00000001:3.0:1775767957.170430:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767957.170432:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767957.170437:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0000. 00000100:00000001:0.0:1775767957.170439:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1775767957.170441:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767957.170447:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aa300 x1862022294256384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775767957.170449:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299883326 00000100:00000010:0.0:1775767957.170461:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942078a3de0. 00000100:00000001:3.0:1775767957.170462:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767957.170469:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767957.170472:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000040:3.0:1775767957.170476:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f33bdf15] to pc [ptlrpcd_rcv+-1] req@ffff9942099aa300 x1862022294256384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767957.170480:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775767957.170492:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767957.170493:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767957.170498:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775767957.170501:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00080000:3.0:1775767957.170503:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775767957.170505:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767957.170507:0:8:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775767957.170512:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767957/real 1775767957] req@ffff9942099aad80 x1862022294256000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767957.170513:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767957.170520:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5209 00000400:00000200:3.0:1775767957.170532:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767957.170540:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767957.170544:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767957.170551:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767957.170554:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000001:0.0:1775767957.170556:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:3.0:1775767957.170557:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000001:0.0:1775767957.170560:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1775767957.170561:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767957.170565:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000400:00000010:0.0:1775767957.170566:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000020:00000040:3.0:1775767957.170567:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:3.0:1775767957.170570:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767957.170574:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000400:00000200:0.0:1775767957.170576:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000020:00000001:3.0:1775767957.170577:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000400:00000010:0.0:1775767957.170584:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000020:00000001:3.0:1775767957.170584:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:3.0:1775767957.170596:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775767957.170597:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1775767957.170599:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767957.170602:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000200:0.0:1775767957.170607:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099aad80 x1862022294256000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767957.170607:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767957.170610:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:3.0:1775767957.170617:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767957.170622:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767957.170627:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767957.170634:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767957.170641:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767957.170648:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767957.170650:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:3.0:1775767957.170654:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1775767957.170655:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099aad80 x1862022294256000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767957.170656:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767957.170660:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8a80. 00000020:00000040:3.0:1775767957.170665:0:8:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:3.0:1775767957.170668:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767957.170670:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767957.170677:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767957.170680:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767957.170684:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767957.170694:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0400. 00000100:00000001:0.0:1775767957.170696:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767957.170698:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767957.170700:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767957.170705:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8a80 x1862022294256512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775767957.170707:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767957.170713:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767957.170721:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767957.170724:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767957.170730:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099aad80 x1862022294256000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775767957.170733:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006c3265af] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8a80 x1862022294256512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767957.170777:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767957.170780:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767957.170787:0:8:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5219) 00000100:00000040:0.0:1775767957.170788:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099aad80 x1862022294256000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767982 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767957.170801:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775767957.170806:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767957.170808:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767957.170810:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767957.170814:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775767957.170817:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775767957.170821:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767957.170826:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775767957.170829:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775767957.170834:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099aad80 x1862022294256000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767982 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767957.170843:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099aad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294256000:192.168.202.156@tcp:250:kworker.0 00000100:00000001:0.0:1775767957.170849:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767957.170851:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767957.170854:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099aad80 x1862022294256000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767982 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767957.170863:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775767957.170866:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775767957.170869:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002b800. 02000000:00000001:0.0:1775767957.170873:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775767957.170876:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775767957.170878:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:0.0:1775767957.170881:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775767957.170884:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b2000. 02000000:00000001:0.0:1775767957.170889:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775767957.170891:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775767957.170893:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099aad80. 00000100:00000001:0.0:1775767957.170898:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775767957.170899:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767957.170902:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767957.170910:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767957.170912:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767957.170915:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775767957.170918:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767957.170919:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775767957.170924:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099aa680 x1862022294256128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767957.170932:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767957.170934:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767957.170936:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775767957.170939:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775767957.170941:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767957.170944:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099aa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294256128:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775767957.170948:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775767957.170950:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775767957.170952:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767957.170954:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775767957.170958:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002b800. 02000000:00000001:0.0:1775767957.170960:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767957.170965:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775767957.170968:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775767957.170972:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775767957.170975:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294256128, portal 10 00000100:00000001:0.0:1775767957.170978:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775767957.170979:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640862336 : -112966068689280 : ffff9942099aa680) 00000100:00000040:0.0:1775767957.170986:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099aa680 x1862022294256128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767982 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767957.170993:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775767957.170995:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775767957.170997:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775767957.170999:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294256128, offset 0 00000400:00000200:0.0:1775767957.171003:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775767957.171008:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767957.171012:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775767957.171015:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775767957.171017:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775767957.171020:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775767957.171023:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775767957.171026:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225174:225174:256:4294967295] 192.168.202.156@tcp LPNI seq info [225170:225170:8:4294967295] 00000400:00000200:0.0:1775767957.171031:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767957.171038:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775767957.171043:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767957.171047:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775767957.171051:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775767957.171053:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775767957.171059:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767957.171062:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1775767957.171066:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767957.171068:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775767957.171071:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775767957.171074:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775767957.171077:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775767957.171079:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775767957.171082:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775767957.171085:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099aa680 x1862022294256128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767982 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767957.171092:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767957.171096:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099aa680 x1862022294256128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767982 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767957.171102:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767957.171104:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775767957.171106:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767957.171110:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:0.0:1775767957.171113:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775767957.171115:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299882003 00000100:00000001:0.0:1775767957.171118:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299883326 00000100:00000001:0.0:1775767957.171121:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299882003 00000100:00000001:0.0:1775767957.171123:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299883326 00000100:00000001:0.0:1775767957.171125:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775767957.171127:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767957.171129:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775767957.171132:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767957/real 1775767957] req@ffff9942099aa680 x1862022294256128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775767957.171140:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775767957.171141:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775767957.171143:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775767957.171145:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775767957.171147:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775767957.171149:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775767957.171151:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099aa680 x1862022294256128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775767957.171187:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099aa680 x1862022294256128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767957.171194:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775767957.171195:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767957.171197:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767957.171199:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775767957.171203:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099aa680 x1862022294256128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767957.171211:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775767957.171215:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099aa680 x1862022294256128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767982 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767957.171223:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775767957.171226:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767957.171227:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767957.171229:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767957.171231:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775767957.171235:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775767957.171238:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767957.171243:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775767957.171245:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775767957.171250:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099aa680 x1862022294256128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767982 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767957.171259:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099aa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294256128:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775767957.171264:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775767957.171269:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ab800 x1862022294256256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767957.171277:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767957.171279:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767957.171281:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775767957.171283:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775767957.171285:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767957.171288:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294256256:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775767957.171292:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775767957.171294:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775767957.171296:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767957.171298:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775767957.171301:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220029400. 02000000:00000001:0.0:1775767957.171304:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767957.171322:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775767957.171325:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775767957.171329:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775767957.171332:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294256256, portal 10 00000100:00000001:0.0:1775767957.171334:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775767957.171335:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640866816 : -112966068684800 : ffff9942099ab800) 00000100:00000040:0.0:1775767957.171341:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ab800 x1862022294256256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767982 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767957.171346:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775767957.171348:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775767957.171349:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775767957.171351:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294256256, offset 0 00000400:00000200:0.0:1775767957.171353:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775767957.171356:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767957.171359:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775767957.171361:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775767957.171363:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775767957.171365:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775767957.171366:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775767957.171369:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225175:225175:256:4294967295] 192.168.202.156@tcp LPNI seq info [225171:225171:8:4294967295] 00000400:00000200:0.0:1775767957.171371:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767957.171376:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775767957.171379:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767957.171381:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775767957.171384:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775767957.171386:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775767957.171389:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767957.171390:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1775767957.171393:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767957.171395:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775767957.171397:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775767957.171399:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775767957.171402:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775767957.171403:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775767957.171406:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775767957.171409:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ab800 x1862022294256256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767982 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767957.171418:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767957.171421:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ab800 x1862022294256256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767982 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767957.171434:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767957.171436:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775767957.171440:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767957.171443:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:0.0:1775767957.171447:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775767957.171449:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299881996 00000100:00000001:0.0:1775767957.171452:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299883326 00000100:00000001:0.0:1775767957.171455:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299881996 00000100:00000001:0.0:1775767957.171457:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299883326 00000100:00000001:0.0:1775767957.171459:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775767957.171461:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767957.171463:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775767957.171467:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767957/real 1775767957] req@ffff9942099ab800 x1862022294256256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775767957.171475:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775767957.171477:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775767957.171478:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775767957.171480:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775767957.171482:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775767957.171485:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775767957.171487:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ab800 x1862022294256256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775767957.171496:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ab800 x1862022294256256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767957.171503:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775767957.171505:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767957.171507:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767957.171508:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775767957.171511:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ab800 x1862022294256256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767957.171520:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775767957.171524:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ab800 x1862022294256256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767982 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767957.171532:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775767957.171535:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767957.171537:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767957.171538:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767957.171539:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775767957.171542:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775767957.171545:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767957.171548:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775767957.171550:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775767957.171554:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ab800 x1862022294256256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767982 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767957.171562:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294256256:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775767957.171565:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775767957.171569:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099aa300 x1862022294256384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767957.171576:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767957.171577:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767957.171580:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775767957.171582:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775767957.171584:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767957.171586:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099aa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294256384:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775767957.171589:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775767957.171591:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775767957.171594:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767957.171595:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775767957.171599:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220028400. 02000000:00000001:0.0:1775767957.171600:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767957.171604:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775767957.171606:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775767957.171610:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775767957.171613:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294256384, portal 4 00000100:00000001:0.0:1775767957.171614:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775767957.171616:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640861440 : -112966068690176 : ffff9942099aa300) 00000100:00000040:0.0:1775767957.171622:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099aa300 x1862022294256384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767982 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767957.171628:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775767957.171630:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775767957.171633:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775767957.171635:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294256384, offset 0 00000400:00000200:0.0:1775767957.171639:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775767957.171643:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767957.171647:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775767957.171649:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775767957.171652:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775767957.171654:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775767957.171657:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775767957.171660:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225176:225176:256:4294967295] 192.168.202.156@tcp LPNI seq info [225172:225172:8:4294967295] 00000400:00000200:0.0:1775767957.171665:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767957.171672:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775767957.171677:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767957.171680:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775767957.171685:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775767957.171688:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775767957.171693:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767957.171696:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1775767957.171700:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767957.171703:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775767957.171706:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775767957.171709:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775767957.171712:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775767957.171714:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775767957.171717:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775767957.171720:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099aa300 x1862022294256384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767982 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767957.171729:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767957.171732:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099aa300 x1862022294256384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767982 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767957.171740:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767957.171742:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775767957.171745:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767957.171748:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:0.0:1775767957.171752:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775767957.171754:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299882019 00000100:00000001:0.0:1775767957.171756:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299883326 00000100:00000001:0.0:1775767957.171759:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299882019 00000100:00000001:0.0:1775767957.171761:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299883326 00000100:00000001:0.0:1775767957.171764:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775767957.171766:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767957.171768:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775767957.171771:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767957/real 1775767957] req@ffff9942099aa300 x1862022294256384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775767957.171780:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775767957.171781:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775767957.171783:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775767957.171785:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775767957.171786:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775767957.171789:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775767957.171792:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099aa300 x1862022294256384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775767957.171801:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099aa300 x1862022294256384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767957.171808:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775767957.171810:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767957.171812:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767957.171813:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775767957.171816:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099aa300 x1862022294256384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767957.171823:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775767957.171827:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099aa300 x1862022294256384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767982 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767957.171834:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775767957.171836:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767957.171838:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767957.171839:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767957.171841:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775767957.171845:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775767957.171847:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767957.171850:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775767957.171851:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775767957.171854:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099aa300 x1862022294256384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767982 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767957.171860:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099aa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294256384:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775767957.171862:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775767957.171865:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a8a80 x1862022294256512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767957.171870:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767957.171871:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767957.171873:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775767957.171874:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775767957.171876:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767957.171877:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294256512:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775767957.171879:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775767957.171881:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775767957.171882:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767957.171883:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775767957.171886:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220028000. 02000000:00000001:0.0:1775767957.171887:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767957.171889:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775767957.171891:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775767957.171893:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775767957.171895:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294256512, portal 4 00000100:00000001:0.0:1775767957.171897:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775767957.171898:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640855168 : -112966068696448 : ffff9942099a8a80) 00000100:00000040:0.0:1775767957.171902:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a8a80 x1862022294256512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767982 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767957.171906:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775767957.171908:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775767957.171909:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775767957.171910:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294256512, offset 0 00000400:00000200:0.0:1775767957.171913:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775767957.171916:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767957.171918:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775767957.171919:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775767957.171921:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775767957.171923:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775767957.171924:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775767957.171927:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225177:225177:256:4294967295] 192.168.202.156@tcp LPNI seq info [225173:225173:8:4294967295] 00000400:00000200:0.0:1775767957.171930:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767957.171934:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775767957.171937:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767957.171939:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775767957.171942:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775767957.171944:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775767957.171947:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767957.171949:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1775767957.171951:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767957.171953:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775767957.171955:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775767957.171957:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775767957.171959:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775767957.171960:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775767957.171961:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775767957.171963:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a8a80 x1862022294256512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767982 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767957.171969:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767957.171971:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a8a80 x1862022294256512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767982 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767957.171975:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767957.171976:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775767957.171978:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767957.171980:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:0.0:1775767957.171981:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775767957.171983:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299882022 00000100:00000001:0.0:1775767957.171984:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299883326 00000100:00000001:0.0:1775767957.171986:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299882022 00000100:00000001:0.0:1775767957.171987:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299883326 00000100:00000001:0.0:1775767957.171989:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775767957.171990:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767957.171992:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775767957.171994:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767957/real 1775767957] req@ffff9942099a8a80 x1862022294256512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775767957.172000:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775767957.172000:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775767957.172001:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775767957.172002:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775767957.172003:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775767957.172005:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775767957.172007:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a8a80 x1862022294256512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775767957.172012:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a8a80 x1862022294256512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767957.172016:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775767957.172017:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767957.172018:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767957.172019:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775767957.172021:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a8a80 x1862022294256512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767957.172025:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775767957.172027:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a8a80 x1862022294256512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767982 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767957.172031:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775767957.172032:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767957.172033:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767957.172034:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767957.172035:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775767957.172038:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775767957.172039:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767957.172042:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775767957.172043:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775767957.172046:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a8a80 x1862022294256512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767982 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767957.172051:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294256512:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775767957.172053:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767957.172054:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767957.172056:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099aa680 x1862022294256128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767982 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767957.172060:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775767957.172061:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775767957.172062:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002b800. 02000000:00000001:0.0:1775767957.172063:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775767957.172064:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775767957.172065:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:0.0:1775767957.172066:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775767957.172067:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b0c00. 02000000:00000001:0.0:1775767957.172070:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775767957.172071:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775767957.172072:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099aa680. 00000100:00000001:0.0:1775767957.172074:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775767957.172075:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767957.172076:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767957.172078:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ab800 x1862022294256256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767982 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767957.172082:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775767957.172083:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775767957.172084:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220029400. 02000000:00000001:0.0:1775767957.172085:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775767957.172086:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775767957.172086:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:0.0:1775767957.172087:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775767957.172089:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b2800. 02000000:00000001:0.0:1775767957.172090:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775767957.172091:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775767957.172092:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ab800. 00000100:00000001:0.0:1775767957.172094:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775767957.172095:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767957.172095:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767957.172097:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099aa300 x1862022294256384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767982 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767957.172101:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775767957.172101:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775767957.172102:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220028400. 02000000:00000001:0.0:1775767957.172103:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775767957.172104:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775767957.172105:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:0.0:1775767957.172106:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775767957.172107:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b0000. 02000000:00000001:0.0:1775767957.172108:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775767957.172108:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775767957.172109:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099aa300. 00000100:00000001:0.0:1775767957.172110:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775767957.172111:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767957.172112:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767957.172113:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a8a80 x1862022294256512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767982 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767957.172117:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775767957.172118:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775767957.172119:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220028000. 02000000:00000001:0.0:1775767957.172120:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775767957.172120:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775767957.172121:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:0.0:1775767957.172122:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775767957.172123:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b0400. 02000000:00000001:0.0:1775767957.172124:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775767957.172124:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775767957.172125:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a8a80. 00000100:00000001:0.0:1775767957.172127:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775767957.172128:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767957.172129:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767957.172132:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767957.172133:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767957.172135:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775767957.172136:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767957.233281:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767957.233287:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767957.233292:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767957.233293:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767957.233296:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767957.233297:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767957.233300:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767957.233301:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767957.233304:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767957.233305:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767957.233363:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767957.233365:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767957.233367:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767957.297289:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767957.297296:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767957.297299:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767957.297305:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767957.297340:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767957.297345:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767957.297348:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767957.297355:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767957.297359:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767958.193272:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767958.193275:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767958.193280:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767958.193282:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767958.193282:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767958.193285:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767958.193291:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:1.0:1775767958.193292:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775767958.193292:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767958.193296:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767958.193312:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:2.0:1775767958.193312:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:0.0:1775767958.193315:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767958.193318:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767958.193319:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767958.193325:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767958.193326:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767958.193326:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767958.193334:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767958.193334:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767958.193338:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767958.193341:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767958.193343:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767958.193346:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767958.193352:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767958.193355:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767958.193358:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767958.257281:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767958.257286:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767958.257290:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767958.257292:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767958.257294:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767958.257295:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767958.257316:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767958.257317:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767958.257320:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767958.257321:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767958.257349:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767958.257351:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767958.257354:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767958.321241:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767958.321257:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767958.321259:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767958.321263:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767958.321266:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767958.321270:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767958.321271:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767958.321274:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767958.321276:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767958.513219:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299883326, at 4299883328 00000100:00000040:0.0:1775767958.513225:0:14953:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4299883326, at 4299883328 00000100:00000010:1.0:1775767958.513230:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3de0. 00000100:00000010:0.0:1775767958.513234:0:14953:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3d80. 00000100:00000001:1.0:1775767959.217283:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767959.217303:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767959.217305:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767959.217306:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767959.217309:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767959.217310:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767959.217311:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767959.217312:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767959.217315:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767959.217326:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:2.0:1775767959.217326:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:1.0:1775767959.217327:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767959.217339:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767959.217339:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767959.217340:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767959.217349:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767959.217350:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767959.217350:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767959.217352:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767959.217356:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767959.217360:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767959.217362:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767959.217366:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767959.217371:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767959.217371:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767959.217376:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767959.217379:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767959.281248:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767959.281254:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767959.281260:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767959.281261:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767959.281265:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767959.281267:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767959.281270:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767959.281272:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767959.281275:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767959.281276:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767959.281306:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767959.281309:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767959.281312:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767959.345244:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767959.345249:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767959.345252:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767959.345256:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767959.345260:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767959.345264:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767959.345266:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767959.345269:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767959.345272:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767960.241318:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767960.241318:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767960.241319:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767960.241326:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767960.241328:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767960.241329:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767960.241335:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767960.241338:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:2.0:1775767960.241338:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775767960.241339:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767960.241345:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767960.241348:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767960.241348:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767960.241355:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767960.241355:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767960.241356:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767960.241365:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767960.241366:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767960.241371:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767960.241372:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767960.241379:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767960.241380:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767960.241391:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767960.241395:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767960.241395:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767960.241400:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767960.241405:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767960.305289:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767960.305296:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767960.305302:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767960.305303:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767960.305308:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767960.305309:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767960.305313:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767960.305315:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767960.305319:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767960.305320:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767960.305341:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767960.305343:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767960.305345:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767960.369268:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767960.369274:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767960.369275:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767960.369292:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767960.369295:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767960.369298:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767960.369300:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767960.369303:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767960.369305:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767961.265226:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767961.265232:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767961.265235:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767961.265238:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767961.265239:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:3.0:1775767961.265239:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767961.265245:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767961.265245:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767961.265245:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767961.265248:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767961.265248:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767961.265249:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767961.265254:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00100000:1.0:1775767961.265255:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:2.0:1775767961.265255:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767961.265258:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767961.265260:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767961.265261:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767961.265262:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767961.265264:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767961.265265:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767961.265284:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767961.265286:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767961.265288:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767961.265290:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767961.265293:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767961.265295:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767961.329307:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767961.329318:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775767961.329321:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775767961.329326:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767961.329332:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775767961.329334:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767961.329336:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767961.329338:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767961.329342:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767961.329343:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767961.329346:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767961.329347:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767961.329351:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767961.329352:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767961.329354:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767961.329355:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767961.329398:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767961.329400:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767961.329403:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767961.393331:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767961.393339:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767961.393341:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767961.393347:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767961.393351:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767961.393355:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767961.393358:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767961.393361:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767961.393363:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767962.289290:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767962.289297:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767962.289299:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767962.289307:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767962.289311:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767962.289316:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767962.289319:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767962.289323:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767962.289325:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767962.289326:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767962.289328:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767962.289331:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767962.289335:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767962.289339:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767962.289342:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767962.289342:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767962.289347:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:2.0:1775767962.289350:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767962.289352:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767962.289359:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767962.289362:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767962.289362:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767962.289367:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767962.289369:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767962.289374:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767962.289377:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1775767962.289378:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775767962.289382:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767962.289384:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775767962.289388:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775767962.289392:0:8:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775767962.289397:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775767962.289404:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5214 00000400:00000200:1.0:1775767962.289417:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767962.289429:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775767962.289432:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775767962.289435:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767962.289439:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767962.289442:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000080:00000001:0.0:1775767962.289443:0:13471:0:(rw.c:1481:ll_writepages()) Process entered 00000100:00000001:1.0:1775767962.289444:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775767962.289449:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000080:00000001:0.0:1775767962.289451:0:13471:0:(file.c:5250:cl_sync_file_range()) Process entered 00000020:00000040:1.0:1775767962.289451:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:1.0:1775767962.289456:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000001:0.0:1775767962.289457:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000040:1.0:1775767962.289459:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:1.0:1775767962.289462:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:1.0:1775767962.289463:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:1.0:1775767962.289465:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767962.289468:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767962.289469:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000010:0.0:1775767962.289470:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff9942103bed80. 00000100:00000001:1.0:1775767962.289471:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775767962.289476:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:1.0:1775767962.289479:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000020:00000010:0.0:1775767962.289480:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0f200. 00000100:00000001:1.0:1775767962.289480:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767962.289485:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775767962.289488:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1775767962.289490:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210b0f600. 02000000:00000001:1.0:1775767962.289490:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767962.289495:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000020:00000010:0.0:1775767962.289496:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942078a3d80. 00000100:00000001:1.0:1775767962.289497:0:8:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:0.0:1775767962.289506:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff9942086e6468. 00000100:00000010:1.0:1775767962.289507:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207888700. 00000020:00000040:1.0:1775767962.289512:0:8:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 00020000:00000010:0.0:1775767962.289513:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994209592f00. 02000000:00000001:1.0:1775767962.289516:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775767962.289518:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:0.0:1775767962.289520:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff99422035b880. 00000100:00000001:1.0:1775767962.289521:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775767962.289525:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000080:00000010:0.0:1775767962.289527:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994207e3e150. 02000000:00000001:1.0:1775767962.289530:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1775767962.289532:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b2238. 00000020:00000010:0.0:1775767962.289536:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0e000. 00000008:00000010:0.0:1775767962.289541:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff99420906be40. 02000000:00000010:1.0:1775767962.289541:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543e800. 00020000:00000010:0.0:1775767962.289546:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff9942103ccfc0. 00008000:00000010:0.0:1775767962.289550:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076afa0. 00000100:00000001:1.0:1775767962.289550:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1775767962.289556:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942107690f0. 00000020:00000001:0.0:1775767962.289559:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107752095112 : -112965957456504 : ffff9942103bed88) 00000020:00001000:0.0:1775767962.289562:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff9942103bed80 00000100:00100000:1.0:1775767962.289562:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207888700 x1862022294256640/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000001:0.0:1775767962.289570:0:13471:0:(cl_object.c:151:cl_object_top()) ffff99420791a5f8 -> ffff99420791a5f8 00000020:00000001:0.0:1775767962.289573:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000100:00000001:1.0:1775767962.289574:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000080:00000001:0.0:1775767962.289576:0:13471:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:0.0:1775767962.289578:0:13471:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x10176:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000100:00000001:1.0:1775767962.289581:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767962.289583:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1775767962.289587:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00000040:1.0:1775767962.289587:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009f2ba301] to pc [ptlrpcd_rcv+-1] req@ffff994207888700 x1862022294256640/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000001:0.0:1775767962.289588:0:13471:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1775767962.289594:0:13471:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x10176:0x0]io ffff9942077b2298 type 6 ignore/verify layout 1/0 00020000:00000002:0.0:1775767962.289602:0:13471:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207dfb6c8) owner 0000000000000000/ffff994220080000 00020000:00000001:0.0:1775767962.289605:0:13471:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1775767962.289607:0:13471:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1775767962.289610:0:13471:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00020000:00000001:0.0:1775767962.289611:0:13471:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767962.289613:0:13471:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1775767962.289614:0:13471:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1775767962.289615:0:13471:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767962.289617:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1775767962.289618:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00000001:1.0:1775767962.289618:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767962.289619:0:13471:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00020000:00000002:0.0:1775767962.289621:0:13471:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207dfb6c8) owner 0000000000000000/ffff994220080000 00000100:00080000:1.0:1775767962.289623:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000001:0.0:1775767962.289624:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767962.289629:0:13471:0:(cl_io.c:823:cl_io_loop()) Process entered 00000100:00080000:1.0:1775767962.289629:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000020:00000001:0.0:1775767962.289631:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000100:00000001:1.0:1775767962.289632:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00020000:00000001:0.0:1775767962.289633:0:13471:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00000100:00080000:1.0:1775767962.289634:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00020000:00200000:0.0:1775767962.289637:0:13471:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00000001:1.0:1775767962.289638:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775767962.289643:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5214 00020000:00000002:0.0:1775767962.289644:0:13471:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00000400:00000200:1.0:1775767962.289649:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:0.0:1775767962.289651:0:13471:0:(lov_io.c:155:lov_sub_get()) Process entered 00000400:00000200:1.0:1775767962.289653:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00020000:00000001:0.0:1775767962.289654:0:13471:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000100:00000001:2.0:1775767962.289657:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1775767962.289658:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00080000:1.0:1775767962.289658:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000020:00000010:0.0:1775767962.289668:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff9942103be000. 00000100:00000040:1.0:1775767962.289669:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767962.289669:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000010:0.0:1775767962.289672:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0ec00. 00000100:00000001:2.0:1775767962.289676:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1775767962.289677:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000010:0.0:1775767962.289679:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210b0ea00. 00000100:00000040:1.0:1775767962.289682:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000010:0.0:1775767962.289683:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942078a3ea0. 00000100:00000001:1.0:1775767962.289690:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000008:00000010:0.0:1775767962.289691:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff9942086e4e18. 00000020:00000001:1.0:1775767962.289692:0:8:0:(genops.c:811:class_conn2export()) Process entered 00020000:00000010:0.0:1775767962.289694:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994209592e40. 00000020:00000040:1.0:1775767962.289696:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000100:00000040:2.0:1775767962.289700:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207888700 x1862022294256640/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1775767962.289701:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00008000:00000010:0.0:1775767962.289702:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994220358000. 00000020:00000040:1.0:1775767962.289711:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000080:00000010:0.0:1775767962.289713:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994207e3e3f0. 00000020:00000001:1.0:1775767962.289722:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000080:00000010:0.0:1775767962.289724:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b0ff8. 00000020:00000001:1.0:1775767962.289734:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000010:0.0:1775767962.289735:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0fa00. 00000008:00000010:0.0:1775767962.289746:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff99420906a6e0. 00000100:00000040:1.0:1775767962.289751:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00020000:00000010:0.0:1775767962.289756:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff9942103cc480. 00000100:00000001:2.0:1775767962.289756:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767962.289761:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00008000:00000010:0.0:1775767962.289764:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076adb0. 00000100:00000001:2.0:1775767962.289764:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767962.289769:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000080:00000010:0.0:1775767962.289773:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff994210769870. 00000020:00000001:0.0:1775767962.289780:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107752091656 : -112965957459960 : ffff9942103be008) 00000100:00000001:1.0:1775767962.289780:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 02000000:00000001:2.0:1775767962.289781:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:1.0:1775767962.289788:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 02000000:00000001:2.0:1775767962.289788:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767962.289798:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1775767962.289800:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff9942103be000 00000100:00080000:1.0:1775767962.289802:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000020:00000001:0.0:1775767962.289808:0:13471:0:(cl_object.c:151:cl_object_top()) ffff99420791ae90 -> ffff99420791a5f8 00000100:00100000:2.0:1775767962.289812:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207888700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294256640:192.168.202.156@tcp:250:kworker.0 00000020:00000001:0.0:1775767962.289814:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000100:00000001:1.0:1775767962.289814:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767962.289821:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1775767962.289827:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 02000000:00000001:1.0:1775767962.289828:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1775767962.289829:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000001:0.0:1775767962.289830:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767962.289837:0:13471:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767962.289839:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00020000:00000001:0.0:1775767962.289840:0:13471:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107752153128 : -112965957398488 : ffff9942103cd028) 02000000:00000001:1.0:1775767962.289843:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767962.289844:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767962.289845:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000020:00000001:0.0:1775767962.289853:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767962.289854:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767962.289854:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00020000:00000001:0.0:1775767962.289860:0:13471:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767962.289862:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000020:00000001:0.0:1775767962.289863:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775767962.289866:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1e800. 00000020:00000001:0.0:1775767962.289867:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00000008:00100000:1.0:1775767962.289867:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00020000:00000001:0.0:1775767962.289872:0:13471:0:(lov_io.c:1108:lov_io_lock()) Process entered 00020000:00000001:0.0:1775767962.289877:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000008:00000001:1.0:1775767962.289877:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767962.289878:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767962.289879:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:0.0:1775767962.289882:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000100:00000001:1.0:1775767962.289885:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767962.289887:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767962.289890:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775767962.289891:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207889f80. 00020000:00000001:0.0:1775767962.289892:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767962.289897:0:13471:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1775767962.289898:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000400:00000200:2.0:1775767962.289902:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:0.0:1775767962.289904:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 02000000:00000001:1.0:1775767962.289906:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000020:00000001:0.0:1775767962.289907:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767962.289911:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767962.289912:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767962.289914:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00000100:00000001:1.0:1775767962.289915:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000010:2.0:1775767962.289916:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39c60. 00020000:00000001:0.0:1775767962.289917:0:13471:0:(lov_io.c:1114:lov_io_start()) Process entered 02000000:00000001:1.0:1775767962.289918:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00020000:00000001:0.0:1775767962.289922:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 02000000:00000001:1.0:1775767962.289923:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767962.289924:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00000008:00000001:0.0:1775767962.289928:0:13471:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000008:00000001:0.0:1775767962.289930:0:13471:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 02000000:00000010:1.0:1775767962.289931:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543f800. 00000400:00000010:2.0:1775767962.289933:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f220. 00000100:00000001:1.0:1775767962.289936:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1775767962.289937:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:0.0:1775767962.289939:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:0.0:1775767962.289941:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:2.0:1775767962.289943:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294256640, portal 25 00000008:00000001:0.0:1775767962.289945:0:13471:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000100:00100000:1.0:1775767962.289946:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207889f80 x1862022294256768/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000008:00000001:0.0:1775767962.289953:0:13471:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000100:00000001:2.0:1775767962.289953:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000008:00000001:0.0:1775767962.289961:0:13471:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107754940544 : -112965954611072 : ffff994210675880) 00000100:00000001:2.0:1775767962.289961:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606120192 : -112966103431424 : ffff994207888700) 00000008:00000020:0.0:1775767962.289972:0:13471:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994210675880 ready 0|-|- wr 1|-|+ rd 0|- 10 in flight 00000100:00000001:1.0:1775767962.289972:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775767962.289976:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1775767962.289977:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:0.0:1775767962.289979:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:0.0:1775767962.289982:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775767962.289982:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ed2274f4] to pc [ptlrpcd_rcv+-1] req@ffff994207889f80 x1862022294256768/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:0.0:1775767962.289990:0:13471:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000100:00000040:2.0:1775767962.289990:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207888700 x1862022294256640/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767987 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000020:0.0:1775767962.289999:0:13471:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994218232fc0 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000008:00000001:0.0:1775767962.290026:0:13471:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767962.290032:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1775767962.290038:0:13471:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767962.290038:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000001:0.0:1775767962.290051:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767962.290053:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775767962.290053:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00020000:00000001:0.0:1775767962.290055:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767962.290061:0:13471:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767962.290062:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000040:2.0:1775767962.290066:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000020:00000001:0.0:1775767962.290069:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767962.290070:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000010:2.0:1775767962.290072:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000020:00000001:0.0:1775767962.290074:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00000100:00000001:1.0:1775767962.290082:0:8:0:(import.c:484:import_select_connection()) Process entered 00020000:00000001:0.0:1775767962.290083:0:13471:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00000100:00000200:2.0:1775767962.290084:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294256640, offset 0 00020000:00000001:0.0:1775767962.290089:0:13471:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000100:00080000:1.0:1775767962.290092:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5214 00000020:00000001:0.0:1775767962.290097:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000001:0.0:1775767962.290100:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00020000:00000001:0.0:1775767962.290102:0:13471:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775767962.290109:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:0.0:1775767962.290111:0:13471:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000400:00000200:2.0:1775767962.290113:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000020:00000001:0.0:1775767962.290114:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00000400:00000200:1.0:1775767962.290117:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000020:00000001:0.0:1775767962.290122:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000100:00080000:1.0:1775767962.290124:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00020000:00000001:0.0:1775767962.290125:0:13471:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00020000:00000001:0.0:1775767962.290128:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000040:1.0:1775767962.290132:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000001:0.0:1775767962.290134:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000400:00000200:2.0:1775767962.290134:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:0.0:1775767962.290139:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:0.0:1775767962.290144:0:13471:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767962.290144:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00020000:00000001:0.0:1775767962.290148:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767962.290151:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00000040:1.0:1775767962.290152:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000200:2.0:1775767962.290190:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1775767962.290193:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00020000:00000001:0.0:1775767962.290200:0:13471:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000400:00000200:2.0:1775767962.290200:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000020:00000001:1.0:1775767962.290201:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000001:0.0:1775767962.290205:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000040:1.0:1775767962.290207:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:0.0:1775767962.290209:0:13471:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000400:00000200:2.0:1775767962.290211:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000020:00000001:0.0:1775767962.290214:0:13471:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000001:1.0:1775767962.290217:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000001:0.0:1775767962.290218:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000400:00000200:2.0:1775767962.290219:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000020:00000040:1.0:1775767962.290222:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00020000:00000001:0.0:1775767962.290224:0:13471:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00020000:00000001:0.0:1775767962.290229:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:1.0:1775767962.290236:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:0.0:1775767962.290237:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000400:00000200:2.0:1775767962.290237:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225178:225178:256:4294967295] 192.168.202.156@tcp LPNI seq info [225174:225174:8:4294967295] 00000020:00000001:0.0:1775767962.290242:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00020000:00000001:0.0:1775767962.290251:0:13471:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775767962.290251:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000400:00000200:2.0:1775767962.290253:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:0.0:1775767962.290272:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767962.290276:0:13471:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000100:00000040:1.0:1775767962.290277:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000001:0.0:1775767962.290282:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000020:00000001:0.0:1775767962.290289:0:13471:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767962.290289:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767962.290297:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000200:2.0:1775767962.290298:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000020:00000001:0.0:1775767962.290299:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1775767962.290305:0:13471:0:(lov_io.c:675:lov_io_fini()) Process entered 00000100:00000001:1.0:1775767962.290306:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00020000:00000001:0.0:1775767962.290311:0:13471:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:0.0:1775767962.290319:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000040:1.0:1775767962.290322:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000020:00000001:0.0:1775767962.290323:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1775767962.290331:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff9942103be000 00000800:00000200:2.0:1775767962.290331:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00080000:1.0:1775767962.290333:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000020:00000010:0.0:1775767962.290341:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210b0ea00. 00000100:00000001:1.0:1775767962.290342:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1775767962.290343:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3300. 02000000:00000001:1.0:1775767962.290346:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775767962.290354:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1775767962.290355:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942078a3ea0. 02000000:00000001:1.0:1775767962.290357:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:0.0:1775767962.290364:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff9942086e4e18. 00000800:00000200:2.0:1775767962.290365:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:1.0:1775767962.290367:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775767962.290374:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00020000:00000010:0.0:1775767962.290376:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994209592e40. 00000008:00000001:1.0:1775767962.290378:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:0.0:1775767962.290381:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994220358000. 00000800:00000200:2.0:1775767962.290382:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000100:00000001:1.0:1775767962.290383:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775767962.290392:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff99420788b100. 00000080:00000010:0.0:1775767962.290395:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994207e3e3f0. 00000020:00000040:1.0:1775767962.290397:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000080:00000010:0.0:1775767962.290405:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b0ff8. 00000800:00000200:2.1:1775767962.290406:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 02000000:00000001:1.0:1775767962.290408:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775767962.290412:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1775767962.290417:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0ec00. 00000100:00000001:1.0:1775767962.290417:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000800:00000200:2.1:1775767962.290418:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 02000000:00000001:1.0:1775767962.290422:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775767962.290429:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:0.0:1775767962.290430:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff99420906a6e0. 00000800:00000100:2.0:1775767962.290439:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 02000000:00000010:1.0:1775767962.290440:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543c400. 00020000:00000010:0.0:1775767962.290443:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff9942103cc480. 00000800:00000010:2.0:1775767962.290443:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3300. 00000100:00000001:1.0:1775767962.290446:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:0.0:1775767962.290458:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076adb0. 00000400:00000200:2.0:1775767962.290459:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00100000:1.0:1775767962.290460:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff99420788b100 x1862022294256896/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775767962.290475:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000080:00000010:0.0:1775767962.290476:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff994210769870. 00000400:00000200:2.0:1775767962.290478:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000100:00000001:1.0:1775767962.290481:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1775767962.290487:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0fa00. 00000100:00000040:1.0:1775767962.290488:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e26cf977] to pc [ptlrpcd_rcv+-1] req@ffff99420788b100 x1862022294256896/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1775767962.290491:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000020:00000010:0.0:1775767962.290493:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff9942103be000. 00020000:00000001:0.0:1775767962.290500:0:13471:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00000400:00000010:2.0:1775767962.290501:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00020000:00000001:0.0:1775767962.290505:0:13471:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000100:00000001:1.0:1775767962.290507:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1775767962.290509:0:13471:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000080:00200000:0.0:1775767962.290513:0:13471:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x10176:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00080000:1.0:1775767962.290513:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775767962.290513:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000080:00000001:0.0:1775767962.290525:0:13471:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000100:00080000:1.0:1775767962.290526:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000020:00000001:0.0:1775767962.290530:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000100:00000001:1.0:1775767962.290531:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000200:2.0:1775767962.290531:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207888700 x1862022294256640/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767987 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00001000:0.0:1775767962.290534:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff9942103bed80 00000100:00080000:1.0:1775767962.290535:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000020:00000010:0.0:1775767962.290542:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210b0f600. 00000100:00000001:1.0:1775767962.290543:0:8:0:(import.c:484:import_select_connection()) Process entered 00000020:00000010:0.0:1775767962.290549:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942078a3d80. 00000100:00080000:1.0:1775767962.290553:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5214 00000008:00000010:0.0:1775767962.290562:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff9942086e6468. 00020000:00000010:0.0:1775767962.290571:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994209592f00. 00000400:00000200:1.0:1775767962.290573:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00008000:00000010:0.0:1775767962.290581:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff99422035b880. 00000400:00000200:1.0:1775767962.290587:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000080:00000010:0.0:1775767962.290596:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994207e3e150. 00000100:00080000:1.0:1775767962.290596:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1775767962.290598:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000080:00000010:0.0:1775767962.290601:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b2238. 00000100:00000040:1.0:1775767962.290603:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000040:2.0:1775767962.290607:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207888700 x1862022294256640/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767987 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767962.290608:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000010:0.0:1775767962.290609:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0f200. 00000100:00000040:1.0:1775767962.290618:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000008:00000010:0.0:1775767962.290619:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff99420906be40. 00020000:00000010:0.0:1775767962.290629:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff9942103ccfc0. 00000100:00000001:1.0:1775767962.290631:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00008000:00000010:0.0:1775767962.290638:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076afa0. 00000020:00000001:1.0:1775767962.290639:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775767962.290646:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000080:00000010:0.0:1775767962.290649:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942107690f0. 00000020:00000001:1.0:1775767962.290652:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000010:0.0:1775767962.290659:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0e000. 00000020:00000040:1.0:1775767962.290659:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000100:00000001:2.0:1775767962.290666:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1775767962.290668:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff9942103bed80. 00000020:00000001:1.0:1775767962.290668:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:2.0:1775767962.290671:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000080:00000001:0.0:1775767962.290673:0:13471:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1775767962.290675:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000080:00000001:0.0:1775767962.290679:0:13471:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767962.290681:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767962.290683:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767962.290684:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767962.290695:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000200:2.0:1775767962.290702:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767962.290704:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:2.0:1775767962.290707:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299888446 00000020:00000040:1.0:1775767962.290708:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:1.0:1775767962.290714:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000010:2.0:1775767962.290717:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d6c00. 00000100:00000001:1.0:1775767962.290718:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767962.290729:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775767962.290734:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767962.290735:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 02000000:00000001:1.0:1775767962.290737:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767962.290740:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775767962.290743:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000100:00000001:2.0:1775767962.290744:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299888446 00000008:00000001:1.0:1775767962.290747:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767962.290751:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775767962.290751:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d6cc0. 00000100:00000001:2.0:1775767962.290762:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000010:1.0:1775767962.290763:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207888e00. 00000020:00000040:1.0:1775767962.290772:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00000100:00000001:2.0:1775767962.290774:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767962.290776:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775767962.290780:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767962.290782:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767962.290785:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775767962.290788:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1775767962.290789:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 02000000:00000001:1.0:1775767962.290794:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767962.290801:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767962/real 1775767962] req@ffff994207888700 x1862022294256640/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1775767962.290803:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543c800. 00000100:00000001:1.0:1775767962.290811:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767962.290825:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207888e00 x1862022294257024/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:2.0:1775767962.290893:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1775767962.290895:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000001:2.0:1775767962.290897:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1775767962.290901:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775767962.290901:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39c60. 00000400:00000200:2.0:1775767962.290909:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f220 00000100:00000040:1.0:1775767962.290910:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001812f966] to pc [ptlrpcd_rcv+-1] req@ffff994207888e00 x1862022294257024/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1775767962.290915:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f220. 00000100:00000001:2.0:1775767962.290927:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1775767962.290934:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767962.290936:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:2.0:1775767962.290941:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207888700 x1862022294256640/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767962.290944:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775767962.290950:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775767962.290956:0:8:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775767962.290968:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775767962.290975:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5214 00000100:00000200:2.0:1775767962.290976:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207888700 x1862022294256640/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775767962.290990:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767962.291002:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775767962.291010:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1775767962.291011:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767962.291014:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775767962.291018:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767962.291019:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767962.291025:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775767962.291025:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775767962.291032:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00100000:2.0:1775767962.291038:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207888700 x1862022294256640/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767962.291039:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775767962.291048:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775767962.291053:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:1.0:1775767962.291058:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775767962.291063:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:1.0:1775767962.291070:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000001:2.0:1775767962.291070:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1775767962.291079:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:2.0:1775767962.291083:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207888700 x1862022294256640/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767987 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775767962.291084:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767962.291093:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767962.291099:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767962.291109:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775767962.291117:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00000001:2.0:1775767962.291122:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775767962.291127:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775767962.291128:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767962.291133:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767962.291138:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767962.291138:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1775767962.291147:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1775767962.291148:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767962.291152:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:1.0:1775767962.291184:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1775767962.291184:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:1.0:1775767962.291192:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775767962.291193:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775767962.291200:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775767962.291201:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000008:00100000:1.0:1775767962.291205:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000100:00000040:2.0:1775767962.291209:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207888700 x1862022294256640/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767987 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:1.0:1775767962.291210:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767962.291219:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775767962.291233:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207889880. 00000020:00000040:1.0:1775767962.291239:0:8:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00000100:00100000:2.0:1775767962.291240:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207888700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294256640:192.168.202.156@tcp:250:kworker.0 02000000:00000001:1.0:1775767962.291249:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775767962.291255:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767962.291257:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767962.291272:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775767962.291275:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1775767962.291276:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 02000000:00000001:1.0:1775767962.291278:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767962.291283:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207888700 x1862022294256640/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767987 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1775767962.291284:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543f000. 00000100:00000001:1.0:1775767962.291293:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767962.291294:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767962.291303:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000100:00100000:1.0:1775767962.291308:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207889880 x1862022294257152/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000010:2.0:1775767962.291309:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1e800. 02000000:00000001:2.0:1775767962.291321:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767962.291326:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767962.291329:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:2.0:1775767962.291338:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767962.291343:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543e800. 00000100:00000001:1.0:1775767962.291344:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775767962.291352:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767962.291353:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767962.291356:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000040:1.0:1775767962.291358:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f11856b8] to pc [ptlrpcd_rcv+-1] req@ffff994207889880 x1862022294257152/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1775767962.291359:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207888700. 00000100:00000001:2.0:1775767962.291367:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767962.291372:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767962.291375:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767962.291377:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775767962.291383:0:8:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5224) 00000100:00000001:2.0:1775767962.291386:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767962.291388:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767962.291391:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775767962.291393:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767962.291395:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767962.291400:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207889f80 x1862022294256768/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767962.291408:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767962.291409:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767962.291411:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767962.291413:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767962.291414:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767962.291416:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207889f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294256768:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775767962.291419:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767962.291421:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767962.291422:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767962.291423:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775767962.291426:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1e800. 02000000:00000001:2.0:1775767962.291428:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767962.291431:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767962.291433:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39c60. 00000400:00000010:2.0:1775767962.291436:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f220. 00000100:00000200:2.0:1775767962.291438:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294256768, portal 10 00000100:00000001:2.0:1775767962.291440:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767962.291441:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606126464 : -112966103425152 : ffff994207889f80) 00000100:00000040:2.0:1775767962.291446:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207889f80 x1862022294256768/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767987 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767962.291450:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767962.291452:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767962.291454:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775767962.291456:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294256768, offset 0 00000400:00000200:2.0:1775767962.291459:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767962.291462:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767962.291465:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767962.291467:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767962.291469:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767962.291471:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767962.291473:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767962.291476:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225179:225179:256:4294967295] 192.168.202.156@tcp LPNI seq info [225175:225175:8:4294967295] 00000400:00000200:2.0:1775767962.291479:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767962.291486:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767962.291490:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767962.291493:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3300. 00000800:00000200:2.0:1775767962.291497:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775767962.291501:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775767962.291506:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767962.291509:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1775767962.291513:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767962.291515:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3300. 00000400:00000200:2.0:1775767962.291518:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775767962.291522:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775767962.291526:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775767962.291527:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775767962.291530:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767962.291534:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207889f80 x1862022294256768/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767987 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767962.291543:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767962.291546:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207889f80 x1862022294256768/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767987 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767962.291554:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767962.291555:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775767962.291558:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767962.291562:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:2.0:1775767962.291564:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767962.291566:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299887124 00000100:00000001:2.0:1775767962.291568:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299888446 00000100:00000001:2.0:1775767962.291570:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299887124 00000100:00000001:2.0:1775767962.291572:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299888446 00000100:00000001:2.0:1775767962.291574:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767962.291575:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767962.291576:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767962.291579:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767962/real 1775767962] req@ffff994207889f80 x1862022294256768/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775767962.291584:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767962.291585:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767962.291587:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39c60. 00000400:00000200:2.0:1775767962.291589:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f220 00000400:00000010:2.0:1775767962.291590:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f220. 00000100:00000001:2.0:1775767962.291591:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767962.291593:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207889f80 x1862022294256768/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767962.291599:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207889f80 x1862022294256768/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767962.291604:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767962.291604:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767962.291606:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767962.291606:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767962.291608:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207889f80 x1862022294256768/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767962.291612:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767962.291615:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207889f80 x1862022294256768/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767987 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767962.291619:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767962.291620:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767962.291621:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767962.291622:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767962.291623:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767962.291625:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767962.291627:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767962.291629:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767962.291630:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767962.291633:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207889f80 x1862022294256768/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767987 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767962.291637:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207889f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294256768:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775767962.291640:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767962.291643:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff99420788b100 x1862022294256896/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767962.291648:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767962.291649:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767962.291651:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767962.291652:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767962.291653:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767962.291655:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff99420788b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294256896:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775767962.291657:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767962.291659:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767962.291660:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767962.291662:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775767962.291664:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f000. 02000000:00000001:2.0:1775767962.291666:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767962.291668:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767962.291670:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39c60. 00000400:00000010:2.0:1775767962.291672:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f220. 00000100:00000200:2.0:1775767962.291674:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294256896, portal 10 00000100:00000001:2.0:1775767962.291675:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767962.291676:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606130944 : -112966103420672 : ffff99420788b100) 00000100:00000040:2.0:1775767962.291680:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff99420788b100 x1862022294256896/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767987 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767962.291685:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767962.291686:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767962.291688:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775767962.291690:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294256896, offset 0 00000400:00000200:2.0:1775767962.291692:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767962.291694:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767962.291697:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767962.291698:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767962.291701:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767962.291702:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767962.291704:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767962.291706:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225180:225180:256:4294967295] 192.168.202.156@tcp LPNI seq info [225176:225176:8:4294967295] 00000400:00000200:2.0:1775767962.291709:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767962.291713:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767962.291716:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767962.291718:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3300. 00000800:00000200:2.0:1775767962.291720:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775767962.291722:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775767962.291725:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767962.291727:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1775767962.291730:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767962.291731:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3300. 00000400:00000200:2.0:1775767962.291733:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775767962.291735:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775767962.291737:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775767962.291739:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775767962.291740:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767962.291743:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff99420788b100 x1862022294256896/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767987 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767962.291748:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767962.291750:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff99420788b100 x1862022294256896/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767987 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767962.291755:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767962.291756:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775767962.291757:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767962.291759:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:2.0:1775767962.291761:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767962.291762:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299887116 00000100:00000001:2.0:1775767962.291763:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299888446 00000100:00000001:2.0:1775767962.291765:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299887116 00000100:00000001:2.0:1775767962.291767:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299888446 00000100:00000001:2.0:1775767962.291769:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767962.291770:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767962.291771:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767962.291774:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767962/real 1775767962] req@ffff99420788b100 x1862022294256896/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775767962.291778:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767962.291779:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767962.291780:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39c60. 00000400:00000200:2.0:1775767962.291781:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f220 00000400:00000010:2.0:1775767962.291782:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f220. 00000100:00000001:2.0:1775767962.291783:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767962.291785:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff99420788b100 x1862022294256896/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767962.291789:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff99420788b100 x1862022294256896/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767962.291794:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767962.291795:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767962.291796:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767962.291797:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767962.291800:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff99420788b100 x1862022294256896/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767962.291803:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767962.291806:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff99420788b100 x1862022294256896/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767987 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767962.291810:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767962.291811:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767962.291812:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767962.291813:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767962.291814:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767962.291817:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767962.291819:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767962.291821:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767962.291823:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767962.291826:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff99420788b100 x1862022294256896/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767987 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767962.291831:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff99420788b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294256896:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775767962.291833:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767962.291836:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207888e00 x1862022294257024/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767962.291840:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767962.291841:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767962.291842:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767962.291843:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767962.291844:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767962.291846:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207888e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294257024:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767962.291848:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767962.291849:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767962.291850:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767962.291851:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775767962.291853:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1ec00. 02000000:00000001:2.0:1775767962.291854:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767962.291856:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767962.291857:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39c60. 00000400:00000010:2.0:1775767962.291859:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f220. 00000100:00000200:2.0:1775767962.291861:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294257024, portal 4 00000100:00000001:2.0:1775767962.291863:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767962.291864:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606121984 : -112966103429632 : ffff994207888e00) 00000100:00000040:2.0:1775767962.291867:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207888e00 x1862022294257024/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767987 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767962.291874:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767962.291876:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767962.291879:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775767962.291881:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294257024, offset 0 00000400:00000200:2.0:1775767962.291886:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767962.291889:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767962.291893:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767962.291895:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767962.291898:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767962.291900:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767962.291902:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767962.291906:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225181:225181:256:4294967295] 192.168.202.156@tcp LPNI seq info [225177:225177:8:4294967295] 00000400:00000200:2.0:1775767962.291910:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767962.291917:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767962.291922:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767962.291926:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3300. 00000800:00000200:2.0:1775767962.291930:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775767962.291933:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775767962.291937:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767962.291940:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1775767962.291943:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767962.291945:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3300. 00000400:00000200:2.0:1775767962.291947:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775767962.291950:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775767962.291952:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775767962.291953:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775767962.291955:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767962.291957:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207888e00 x1862022294257024/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767987 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767962.291963:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767962.291965:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207888e00 x1862022294257024/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767987 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767962.291974:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767962.291975:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775767962.291977:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767962.291980:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:2.0:1775767962.291982:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767962.291984:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299887139 00000100:00000001:2.0:1775767962.291986:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299888446 00000100:00000001:2.0:1775767962.291988:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299887139 00000100:00000001:2.0:1775767962.291989:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299888446 00000100:00000001:2.0:1775767962.291990:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767962.291992:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767962.291993:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767962.291995:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767962/real 1775767962] req@ffff994207888e00 x1862022294257024/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775767962.292000:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767962.292001:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767962.292002:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39c60. 00000400:00000200:2.0:1775767962.292004:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f220 00000400:00000010:2.0:1775767962.292005:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f220. 00000100:00000001:2.0:1775767962.292007:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767962.292009:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207888e00 x1862022294257024/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767962.292014:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207888e00 x1862022294257024/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767962.292018:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767962.292019:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767962.292020:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767962.292021:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767962.292023:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207888e00 x1862022294257024/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767962.292028:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767962.292030:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207888e00 x1862022294257024/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767987 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767962.292035:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767962.292036:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767962.292037:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767962.292038:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767962.292039:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767962.292042:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767962.292043:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767962.292045:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767962.292046:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767962.292049:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207888e00 x1862022294257024/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767987 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767962.292053:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207888e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294257024:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767962.292055:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767962.292058:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207889880 x1862022294257152/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767962.292062:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767962.292064:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767962.292065:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767962.292066:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767962.292068:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767962.292069:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207889880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294257152:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767962.292071:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767962.292073:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767962.292074:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767962.292075:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775767962.292077:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1c800. 02000000:00000001:2.0:1775767962.292079:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767962.292081:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767962.292082:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39c60. 00000400:00000010:2.0:1775767962.292084:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f220. 00000100:00000200:2.0:1775767962.292086:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294257152, portal 4 00000100:00000001:2.0:1775767962.292088:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767962.292089:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606124672 : -112966103426944 : ffff994207889880) 00000100:00000040:2.0:1775767962.292092:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207889880 x1862022294257152/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767987 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767962.292097:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767962.292098:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767962.292100:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775767962.292101:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294257152, offset 0 00000400:00000200:2.0:1775767962.292104:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767962.292106:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767962.292108:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767962.292109:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767962.292111:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767962.292112:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767962.292114:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767962.292117:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225182:225182:256:4294967295] 192.168.202.156@tcp LPNI seq info [225178:225178:8:4294967295] 00000400:00000200:2.0:1775767962.292119:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767962.292123:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767962.292126:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767962.292128:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3300. 00000800:00000200:2.0:1775767962.292131:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775767962.292133:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775767962.292135:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767962.292137:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1775767962.292139:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767962.292141:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3300. 00000400:00000200:2.0:1775767962.292143:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775767962.292145:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775767962.292147:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775767962.292148:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775767962.292149:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767962.292151:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207889880 x1862022294257152/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767987 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767962.292176:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767962.292178:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207889880 x1862022294257152/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767987 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767962.292182:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767962.292183:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775767962.292186:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767962.292188:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:2.0:1775767962.292190:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767962.292191:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299887143 00000100:00000001:2.0:1775767962.292193:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299888446 00000100:00000001:2.0:1775767962.292195:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299887143 00000100:00000001:2.0:1775767962.292196:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299888446 00000100:00000001:2.0:1775767962.292198:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767962.292199:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767962.292200:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767962.292202:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767962/real 1775767962] req@ffff994207889880 x1862022294257152/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775767962.292206:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767962.292207:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767962.292209:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39c60. 00000400:00000200:2.0:1775767962.292210:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f220 00000400:00000010:2.0:1775767962.292211:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f220. 00000100:00000001:2.0:1775767962.292213:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767962.292215:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207889880 x1862022294257152/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767962.292221:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207889880 x1862022294257152/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767962.292225:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767962.292225:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767962.292226:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767962.292227:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767962.292229:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207889880 x1862022294257152/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767962.292233:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767962.292235:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207889880 x1862022294257152/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767987 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767962.292239:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767962.292240:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767962.292241:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767962.292242:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767962.292243:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767962.292246:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767962.292247:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767962.292249:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767962.292251:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767962.292253:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207889880 x1862022294257152/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767987 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767962.292274:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207889880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294257152:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767962.292277:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767962.292279:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767962.292281:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207889f80 x1862022294256768/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767987 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767962.292286:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767962.292287:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767962.292289:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1e800. 02000000:00000001:2.0:1775767962.292290:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767962.292291:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767962.292292:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:2.0:1775767962.292294:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767962.292295:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543f800. 02000000:00000001:2.0:1775767962.292297:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767962.292298:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767962.292300:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207889f80. 00000100:00000001:2.0:1775767962.292302:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767962.292303:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767962.292304:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767962.292306:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff99420788b100 x1862022294256896/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767987 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767962.292311:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767962.292312:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767962.292313:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f000. 02000000:00000001:2.0:1775767962.292314:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767962.292315:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767962.292316:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:2.0:1775767962.292317:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767962.292317:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543c400. 02000000:00000001:2.0:1775767962.292319:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767962.292319:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767962.292320:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff99420788b100. 00000100:00000001:2.0:1775767962.292321:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767962.292322:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767962.292323:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767962.292325:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207888e00 x1862022294257024/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767987 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767962.292329:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767962.292330:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767962.292331:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1ec00. 02000000:00000001:2.0:1775767962.292332:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767962.292333:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767962.292334:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:2.0:1775767962.292335:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767962.292336:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543c800. 02000000:00000001:2.0:1775767962.292337:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767962.292338:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767962.292339:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207888e00. 00000100:00000001:2.0:1775767962.292340:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767962.292341:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767962.292342:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767962.292343:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207889880 x1862022294257152/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767987 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767962.292347:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767962.292348:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767962.292348:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1c800. 02000000:00000001:2.0:1775767962.292350:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767962.292350:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767962.292351:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:2.0:1775767962.292352:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767962.292353:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543f000. 02000000:00000001:2.0:1775767962.292354:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767962.292354:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767962.292355:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207889880. 00000100:00000001:2.0:1775767962.292356:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767962.292357:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767962.292358:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767962.292361:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767962.292362:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767962.292363:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775767962.292364:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767962.353306:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767962.353315:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767962.353324:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767962.353326:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767962.353331:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767962.353333:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767962.353337:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767962.353338:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767962.353342:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767962.353344:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767962.353368:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767962.353370:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767962.353373:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767962.417250:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767962.417272:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767962.417275:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767962.417282:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767962.417287:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767962.417293:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767962.417295:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767962.417299:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767962.417302:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767963.313292:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767963.313299:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767963.313301:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767963.313302:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767963.313308:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:2.0:1775767963.313309:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:0.0:1775767963.313311:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767963.313319:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:0.0:1775767963.313320:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775767963.313326:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767963.313326:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767963.313334:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767963.313334:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767963.313339:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767963.313343:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767963.313346:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767963.313349:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767963.313352:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767963.313368:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767963.313373:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767963.313375:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767963.313379:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767963.313381:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767963.313386:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767963.313389:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767963.313391:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767963.313393:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767963.377276:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767963.377283:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767963.377288:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767963.377289:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767963.377296:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767963.377298:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767963.377301:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767963.377302:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767963.377304:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767963.377305:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767963.377353:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767963.377357:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767963.377361:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767963.441297:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767963.441305:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767963.441306:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767963.441311:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767963.441315:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767963.441319:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767963.441321:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767963.441324:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767963.441328:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767963.633270:0:14953:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4299888446, at 4299888448 00000100:00000040:1.0:1775767963.633270:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299888446, at 4299888448 00000100:00000010:0.0:1775767963.633289:0:14953:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d6c00. 00000100:00000010:1.0:1775767963.633291:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d6cc0. 00000100:00000001:0.0:1775767964.337291:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767964.337291:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767964.337299:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767964.337300:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767964.337301:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767964.337303:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767964.337310:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775767964.337311:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775767964.337316:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767964.337317:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767964.337321:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767964.337321:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767964.337325:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767964.337328:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767964.337331:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767964.337332:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767964.337335:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767964.337338:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767964.337399:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767964.337407:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767964.337410:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767964.337414:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775767964.337418:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767964.337422:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767964.337424:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767964.337427:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767964.337429:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767964.401268:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767964.401277:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767964.401285:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767964.401286:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767964.401291:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767964.401292:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767964.401296:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767964.401298:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767964.401302:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767964.401304:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767964.401327:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767964.401329:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767964.401332:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767964.465210:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767964.465217:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767964.465219:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767964.465225:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767964.465228:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767964.465233:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767964.465247:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767964.465251:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767964.465255:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767965.361298:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767965.361298:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767965.361305:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767965.361308:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767965.361311:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767965.361314:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767965.361314:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775767965.361318:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:2.0:1775767965.361321:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767965.361321:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767965.361328:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767965.361329:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767965.361330:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767965.361330:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767965.361333:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767965.361337:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767965.361337:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775767965.361341:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767965.361343:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767965.361344:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767965.361349:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767965.361349:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767965.361356:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767965.361357:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767965.361366:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767965.361369:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767965.361374:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767965.425306:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767965.425312:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767965.425318:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767965.425320:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767965.425323:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767965.425324:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767965.425327:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767965.425328:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767965.425330:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767965.425331:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767965.425354:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767965.425356:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767965.425358:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767965.489312:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767965.489319:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767965.489321:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767965.489327:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767965.489330:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767965.489341:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767965.489343:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767965.489346:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767965.489350:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767966.385262:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767966.385272:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767966.385276:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767966.385283:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:3.0:1775767966.385283:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767966.385284:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767966.385292:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767966.385296:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767966.385296:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767966.385297:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767966.385303:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767966.385305:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767966.385305:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767966.385308:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767966.385313:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767966.385314:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767966.385318:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767966.385321:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767966.385322:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767966.385323:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767966.385327:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767966.385328:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767966.385332:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767966.385336:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767966.385337:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767966.385340:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767966.385343:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767966.449302:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767966.449310:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767966.449317:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767966.449320:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775767966.449323:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775767966.449327:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767966.449330:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775767966.449332:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767966.449335:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767966.449337:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767966.449342:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767966.449344:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775767966.449346:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775767966.449347:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767966.449349:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775767966.449350:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767966.449352:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767966.449354:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767966.449358:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767966.449359:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775767966.449361:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775767966.449362:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767966.449364:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775767966.449366:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767966.449367:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767966.449369:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767966.449373:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767966.449375:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775767966.449376:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775767966.449378:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767966.449380:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775767966.449381:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767966.449383:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767966.449385:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767966.449413:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767966.449416:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767966.449419:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767966.513233:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767966.513240:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767966.513242:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767966.513247:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767966.513250:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767966.513253:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767966.513255:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767966.513257:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767966.513259:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767967.409270:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767967.409270:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767967.409278:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767967.409278:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767967.409281:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767967.409282:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767967.409286:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:3.0:1775767967.409288:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:2.0:1775767967.409289:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767967.409290:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767967.409294:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767967.409295:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767967.409296:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1775767967.409298:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000080:00000001:0.0:1775767967.409299:0:13471:0:(rw.c:1481:ll_writepages()) Process entered 00000100:00000001:2.0:1775767967.409301:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1775767967.409302:0:13471:0:(file.c:5250:cl_sync_file_range()) Process entered 00000100:00000001:1.0:1775767967.409302:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767967.409302:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767967.409305:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1775767967.409307:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00000001:2.0:1775767967.409308:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767967.409311:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767967.409313:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767967.409314:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000010:0.0:1775767967.409317:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff9942103bed80. 00000100:00000001:3.0:1775767967.409317:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767967.409324:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767967.409324:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1775767967.409325:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0f200. 00000100:00000001:1.0:1775767967.409326:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767967.409331:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1775767967.409333:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210b0f600. 00000020:00000010:0.0:1775767967.409337:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942078a3d80. 00000008:00000010:0.0:1775767967.409343:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff9942086e6468. 00020000:00000010:0.0:1775767967.409346:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994209592f00. 00000100:00080000:1.0:1775767967.409348:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00008000:00000010:0.0:1775767967.409350:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff99422035b880. 00000100:00080000:1.0:1775767967.409353:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775767967.409356:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000080:00000010:0.0:1775767967.409357:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994207e3e150. 00000100:00080000:1.0:1775767967.409359:0:8:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000080:00000010:0.0:1775767967.409361:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b2238. 00000100:00000001:1.0:1775767967.409363:0:8:0:(import.c:484:import_select_connection()) Process entered 00000020:00000010:0.0:1775767967.409366:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0fa00. 00000008:00000010:0.0:1775767967.409370:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff99420906be40. 00000100:00080000:1.0:1775767967.409370:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5219 00020000:00000010:0.0:1775767967.409375:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff9942103ccfc0. 00008000:00000010:0.0:1775767967.409378:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076afa0. 00000080:00000010:0.0:1775767967.409380:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942107690f0. 00000020:00000001:0.0:1775767967.409382:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107752095112 : -112965957456504 : ffff9942103bed88) 00000020:00001000:0.0:1775767967.409385:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff9942103bed80 00000400:00000200:1.0:1775767967.409386:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:0.0:1775767967.409389:0:13471:0:(cl_object.c:151:cl_object_top()) ffff9942078a76c0 -> ffff9942078a76c0 00000020:00000001:0.0:1775767967.409392:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000400:00000200:1.0:1775767967.409393:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000080:00000001:0.0:1775767967.409395:0:13471:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000100:00080000:1.0:1775767967.409397:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000080:00200000:0.0:1775767967.409398:0:13471:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000100:00000040:1.0:1775767967.409403:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000001:0.0:1775767967.409404:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1775767967.409407:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00000001:1.0:1775767967.409408:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000080:00000001:0.0:1775767967.409409:0:13471:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767967.409413:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00020000:00000002:0.0:1775767967.409414:0:13471:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017a:0x0]io ffff9942077b2298 type 6 ignore/verify layout 1/0 00000100:00000001:1.0:1775767967.409415:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00020000:00000002:0.0:1775767967.409419:0:13471:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207c1f9b0) owner 0000000000000000/ffff994220080000 00000020:00000001:1.0:1775767967.409420:0:8:0:(genops.c:811:class_conn2export()) Process entered 00020000:00000001:0.0:1775767967.409424:0:13471:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00000020:00000040:1.0:1775767967.409424:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00020000:00000001:0.0:1775767967.409426:0:13471:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00000020:00000001:1.0:1775767967.409426:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775767967.409428:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00020000:00000001:0.0:1775767967.409429:0:13471:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00020000:00000001:0.0:1775767967.409432:0:13471:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775767967.409433:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00020000:00000001:0.0:1775767967.409434:0:13471:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1775767967.409437:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00020000:00000001:0.0:1775767967.409438:0:13471:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1775767967.409440:0:13471:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767967.409443:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000100:00000040:1.0:1775767967.409444:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000001:0.0:1775767967.409446:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00000001:1.0:1775767967.409447:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00020000:00000001:0.0:1775767967.409448:0:13471:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00000100:00000040:1.0:1775767967.409451:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00020000:00000002:0.0:1775767967.409452:0:13471:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207c1f9b0) owner 0000000000000000/ffff994220080000 00000100:00000001:1.0:1775767967.409454:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775767967.409455:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1775767967.409459:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:1.0:1775767967.409462:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775767967.409463:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767967.409465:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000020:00000001:0.0:1775767967.409468:0:13471:0:(cl_io.c:823:cl_io_loop()) Process entered 02000000:00000001:1.0:1775767967.409468:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1775767967.409472:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 02000000:00000001:1.0:1775767967.409472:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767967.409475:0:13471:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00000100:00000001:1.0:1775767967.409476:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00020000:00200000:0.0:1775767967.409479:0:13471:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00000001:1.0:1775767967.409479:0:8:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1775767967.409483:0:13471:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00000100:00000010:1.0:1775767967.409485:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207889880. 00020000:00000001:0.0:1775767967.409486:0:13471:0:(lov_io.c:155:lov_sub_get()) Process entered 00020000:00000001:0.0:1775767967.409489:0:13471:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000020:00000001:0.0:1775767967.409490:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000040:1.0:1775767967.409490:0:8:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 00000020:00000010:0.0:1775767967.409495:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff9942103be000. 02000000:00000001:1.0:1775767967.409495:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775767967.409498:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1775767967.409499:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0ec00. 00000100:00000001:1.0:1775767967.409501:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000020:00000010:0.0:1775767967.409502:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210b0ea00. 02000000:00000001:1.0:1775767967.409504:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000020:00000010:0.0:1775767967.409505:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942078a3ea0. 02000000:00000001:1.0:1775767967.409508:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:0.0:1775767967.409509:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff9942086e4e18. 00020000:00000010:0.0:1775767967.409513:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994209592e40. 00008000:00000010:0.0:1775767967.409516:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994220358000. 00000080:00000010:0.0:1775767967.409519:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994207e3e3f0. 02000000:00000010:1.0:1775767967.409519:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543fc00. 00000080:00000010:0.0:1775767967.409524:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b0ff8. 00000020:00000010:0.0:1775767967.409526:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0fc00. 00000100:00000001:1.0:1775767967.409526:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:0.0:1775767967.409528:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff99420906a6e0. 00020000:00000010:0.0:1775767967.409531:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff9942103cc480. 00008000:00000010:0.0:1775767967.409533:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076adb0. 00000080:00000010:0.0:1775767967.409536:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff994210769870. 00000100:00100000:1.0:1775767967.409537:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207889880 x1862022294257280/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000001:0.0:1775767967.409540:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107752091656 : -112965957459960 : ffff9942103be008) 00000020:00001000:0.0:1775767967.409544:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff9942103be000 00000020:00000001:0.0:1775767967.409548:0:13471:0:(cl_object.c:151:cl_object_top()) ffff9942078a7788 -> ffff9942078a76c0 00000020:00000001:0.0:1775767967.409551:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:0.0:1775767967.409554:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000100:00000001:1.0:1775767967.409554:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000020:00000001:0.0:1775767967.409555:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:0.0:1775767967.409558:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767967.409560:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767967.409561:0:13471:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767967.409566:0:13471:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107752153128 : -112965957398488 : ffff9942103cd028) 00000100:00000040:1.0:1775767967.409569:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f11856b8] to pc [ptlrpcd_rcv+-1] req@ffff994207889880 x1862022294257280/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775767967.409570:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000020:00000001:0.0:1775767967.409572:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767967.409575:0:13471:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767967.409583:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767967.409585:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00020000:00000001:0.0:1775767967.409586:0:13471:0:(lov_io.c:1108:lov_io_lock()) Process entered 00020000:00000001:0.0:1775767967.409588:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:0.0:1775767967.409588:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:0.0:1775767967.409590:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:0.0:1775767967.409591:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767967.409591:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767967.409595:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767967.409597:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767967.409597:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00020000:00000001:0.0:1775767967.409600:0:13471:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767967.409603:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000100:00080000:1.0:1775767967.409603:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000020:00000001:0.0:1775767967.409605:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767967.409607:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767967.409607:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000020:00000001:0.0:1775767967.409610:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00000100:00080000:1.0:1775767967.409610:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00020000:00000001:0.0:1775767967.409613:0:13471:0:(lov_io.c:1114:lov_io_start()) Process entered 00000100:00000001:1.0:1775767967.409614:0:8:0:(import.c:484:import_select_connection()) Process entered 00020000:00000001:0.0:1775767967.409615:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:0.0:1775767967.409617:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00000100:00080000:1.0:1775767967.409619:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5219 00000008:00000001:0.0:1775767967.409620:0:13471:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000400:00000200:1.0:1775767967.409623:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00000001:0.0:1775767967.409624:0:13471:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000400:00000200:1.0:1775767967.409627:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000008:00000001:0.0:1775767967.409629:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000100:00080000:1.0:1775767967.409629:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775767967.409632:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000008:00000020:0.0:1775767967.409633:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:0.0:1775767967.409636:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767967.409636:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775767967.409637:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00000001:0.0:1775767967.409639:0:13471:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000100:00000001:2.0:1775767967.409641:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000008:00000001:0.0:1775767967.409642:0:13471:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000100:00000040:1.0:1775767967.409642:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767967.409643:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000008:00000001:0.0:1775767967.409645:0:13471:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107884691392 : -112965824860224 : ffff994218232fc0) 00000100:00000001:1.0:1775767967.409646:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775767967.409650:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000008:00000020:0.0:1775767967.409651:0:13471:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994218232fc0 ready 0|-|- wr 1|-|+ rd 0|- 10 in flight 00000100:00000040:2.0:1775767967.409652:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207889880 x1862022294257280/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1775767967.409653:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:1.0:1775767967.409656:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775767967.409659:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:1.0:1775767967.409663:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000008:00000001:0.0:1775767967.409667:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000020:00000001:1.0:1775767967.409668:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000008:00000020:0.0:1775767967.409671:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:0.0:1775767967.409675:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767967.409675:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000040:1.0:1775767967.409676:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767967.409677:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767967.409681:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000008:00000001:0.0:1775767967.409683:0:13471:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 02000000:00000001:2.0:1775767967.409684:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:1.0:1775767967.409685:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000001:2.0:1775767967.409687:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000020:0.0:1775767967.409688:0:13471:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994210675880 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000100:00000001:1.0:1775767967.409688:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 02000000:00000001:2.0:1775767967.409692:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1775767967.409694:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00100000:2.0:1775767967.409700:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207889880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294257280:192.168.202.156@tcp:250:kworker.0 00000008:00000001:0.0:1775767967.409701:0:13471:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1775767967.409701:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000008:00000001:0.0:1775767967.409708:0:13471:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767967.409708:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767967.409709:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775767967.409712:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775767967.409713:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:0.0:1775767967.409714:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767967.409716:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767967.409717:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767967.409718:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767967.409722:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00020000:00000001:0.0:1775767967.409723:0:13471:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767967.409723:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767967.409728:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767967.409728:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000010:2.0:1775767967.409729:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1c800. 00000020:00000001:0.0:1775767967.409733:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00000008:00100000:1.0:1775767967.409733:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1775767967.409734:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767967.409737:0:13471:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00000008:00000001:1.0:1775767967.409739:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767967.409740:0:13471:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000020:00000001:0.0:1775767967.409743:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00000100:00000001:1.0:1775767967.409743:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767967.409744:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:0.0:1775767967.409745:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00000100:00000010:1.0:1775767967.409750:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207888e00. 00000400:00000010:2.0:1775767967.409750:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39c60. 00000020:00000040:1.0:1775767967.409754:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00020000:00000001:0.0:1775767967.409755:0:13471:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767967.409759:0:13471:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 02000000:00000001:1.0:1775767967.409759:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000010:2.0:1775767967.409761:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f220. 02000000:00000001:1.0:1775767967.409762:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767967.409765:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00000100:00000001:1.0:1775767967.409766:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000020:00000001:0.0:1775767967.409769:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 02000000:00000001:1.0:1775767967.409769:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000200:2.0:1775767967.409770:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294257280, portal 25 00020000:00000001:0.0:1775767967.409773:0:13471:0:(lov_io.c:1208:lov_io_unlock()) Process entered 02000000:00000001:1.0:1775767967.409773:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767967.409776:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000001:2.0:1775767967.409776:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000020:00000001:0.0:1775767967.409779:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 02000000:00000010:1.0:1775767967.409779:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543d000. 00000100:00000001:2.0:1775767967.409780:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606124672 : -112966103426944 : ffff994207889880) 00000020:00000001:0.0:1775767967.409781:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:0.0:1775767967.409784:0:13471:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767967.409784:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767967.409789:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767967.409792:0:13471:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000100:00000040:2.0:1775767967.409792:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207889880 x1862022294257280/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767992 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767967.409793:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207888e00 x1862022294257408/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000001:0.0:1775767967.409794:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:0.0:1775767967.409797:0:13471:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000020:00000001:0.0:1775767967.409799:0:13471:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000001:0.0:1775767967.409802:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:0.0:1775767967.409804:0:13471:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00020000:00000001:0.0:1775767967.409806:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:0.0:1775767967.409809:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000001:0.0:1775767967.409811:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00020000:00000001:0.0:1775767967.409813:0:13471:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767967.409818:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767967.409827:0:13471:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000020:00000001:0.0:1775767967.409829:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000100:00000001:2.0:1775767967.409829:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1775767967.409830:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000020:00000001:0.0:1775767967.409833:0:13471:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767967.409835:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767967.409837:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00000100:00000040:2.0:1775767967.409839:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00020000:00000001:0.0:1775767967.409840:0:13471:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:0.0:1775767967.409842:0:13471:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000400:00000010:2.0:1775767967.409844:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000020:00000001:0.0:1775767967.409845:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00000100:00000040:1.0:1775767967.409845:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001812f966] to pc [ptlrpcd_rcv+-1] req@ffff994207888e00 x1862022294257408/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775767967.409847:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1775767967.409851:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff9942103be000 00000100:00000200:2.0:1775767967.409853:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294257280, offset 0 00000020:00000010:0.0:1775767967.409855:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210b0ea00. 00000020:00000010:0.0:1775767967.409863:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942078a3ea0. 00000100:00000001:1.0:1775767967.409866:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:0.0:1775767967.409868:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff9942086e4e18. 00000400:00000200:2.0:1775767967.409868:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00080000:1.0:1775767967.409871:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00020000:00000010:0.0:1775767967.409874:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994209592e40. 00000100:00080000:1.0:1775767967.409877:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00008000:00000010:0.0:1775767967.409879:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994220358000. 00000100:00000001:1.0:1775767967.409880:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000080:00000010:0.0:1775767967.409884:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994207e3e3f0. 00000080:00000010:0.0:1775767967.409886:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b0ff8. 00000020:00000010:0.0:1775767967.409891:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0ec00. 00000100:00080000:1.0:1775767967.409891:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:2.0:1775767967.409893:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00000010:0.0:1775767967.409897:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff99420906a6e0. 00000100:00000001:1.0:1775767967.409899:0:8:0:(import.c:484:import_select_connection()) Process entered 00020000:00000010:0.0:1775767967.409901:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff9942103cc480. 00000100:00080000:1.0:1775767967.409905:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5219 00000400:00000200:2.0:1775767967.409906:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00008000:00000010:0.0:1775767967.409908:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076adb0. 00000080:00000010:0.0:1775767967.409915:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff994210769870. 00000400:00000200:1.0:1775767967.409915:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767967.409917:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000010:0.0:1775767967.409921:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0fc00. 00000400:00000200:1.0:1775767967.409924:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000400:00000200:2.0:1775767967.409925:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000020:00000010:0.0:1775767967.409929:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff9942103be000. 00000100:00080000:1.0:1775767967.409929:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1775767967.409932:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00020000:00000001:0.0:1775767967.409934:0:13471:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00000100:00000040:1.0:1775767967.409938:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000200:2.0:1775767967.409938:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00020000:00000001:0.0:1775767967.409939:0:13471:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000100:00000001:1.0:1775767967.409943:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000080:00000001:0.0:1775767967.409944:0:13471:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000100:00000040:1.0:1775767967.409948:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000200:2.0:1775767967.409948:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225183:225183:256:4294967295] 192.168.202.156@tcp LPNI seq info [225179:225179:8:4294967295] 00000080:00200000:0.0:1775767967.409949:0:13471:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00000001:1.0:1775767967.409954:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000080:00000001:0.0:1775767967.409959:0:13471:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:1.0:1775767967.409959:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000001:0.0:1775767967.409962:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00000040:1.0:1775767967.409963:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000400:00000200:2.0:1775767967.409963:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00001000:0.0:1775767967.409966:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff9942103bed80 00000020:00000001:1.0:1775767967.409967:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000010:0.0:1775767967.409970:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210b0f600. 00000020:00000040:1.0:1775767967.409970:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:1.0:1775767967.409975:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000010:0.0:1775767967.409976:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942078a3d80. 00000020:00000001:1.0:1775767967.409981:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000400:00000200:2.0:1775767967.409981:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000008:00000010:0.0:1775767967.409982:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff9942086e6468. 00020000:00000010:0.0:1775767967.409988:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994209592f00. 00000100:00000040:1.0:1775767967.409989:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00008000:00000010:0.0:1775767967.409992:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff99422035b880. 00000100:00000001:1.0:1775767967.409992:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:2.0:1775767967.409996:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000080:00000010:0.0:1775767967.409997:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994207e3e150. 00000100:00000040:1.0:1775767967.409998:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000080:00000010:0.0:1775767967.410003:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b2238. 00000100:00000001:1.0:1775767967.410003:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000800:00000010:2.0:1775767967.410003:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3300. 00000020:00000010:0.0:1775767967.410008:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0f200. 00000020:00000040:1.0:1775767967.410009:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000008:00000010:0.0:1775767967.410012:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff99420906be40. 00000100:00080000:1.0:1775767967.410015:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00020000:00000010:0.0:1775767967.410017:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff9942103ccfc0. 00000800:00000200:2.0:1775767967.410018:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:1.0:1775767967.410021:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:0.0:1775767967.410023:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076afa0. 02000000:00000001:1.0:1775767967.410024:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000080:00000010:0.0:1775767967.410026:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942107690f0. 02000000:00000001:1.0:1775767967.410028:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:2.1:1775767967.410028:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000010:0.0:1775767967.410029:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0fa00. 00000020:00000010:0.0:1775767967.410035:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff9942103bed80. 02000000:00000001:1.0:1775767967.410035:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767967.410037:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000080:00000001:0.0:1775767967.410039:0:13471:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:1.0:1775767967.410040:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000080:00000001:0.0:1775767967.410042:0:13471:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1775767967.410043:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767967.410046:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767967.410048:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1775767967.410049:0:13471:0:(rw.c:1481:ll_writepages()) Process entered 00000100:00000010:1.0:1775767967.410050:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff99420788b100. 00000080:00000001:0.0:1775767967.410052:0:13471:0:(file.c:5250:cl_sync_file_range()) Process entered 00000020:00000001:0.0:1775767967.410055:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000040:1.0:1775767967.410055:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000400:00000200:2.0:1775767967.410056:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000010:0.0:1775767967.410059:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff9942103bed80. 02000000:00000001:1.0:1775767967.410061:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1775767967.410061:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299893565 00000020:00000010:0.0:1775767967.410062:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0fa00. 02000000:00000001:1.0:1775767967.410063:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767967.410067:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000020:00000010:0.0:1775767967.410068:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210b0f200. 02000000:00000001:1.0:1775767967.410069:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000010:2.0:1775767967.410069:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d6540. 00000020:00000010:0.0:1775767967.410072:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942078a3d80. 02000000:00000001:1.0:1775767967.410072:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767967.410077:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 02000000:00000010:1.0:1775767967.410080:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543e400. 00000100:00000001:2.0:1775767967.410083:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299893565 00000100:00000001:1.0:1775767967.410084:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:0.0:1775767967.410087:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff9942086e6468. 00000100:00000010:2.0:1775767967.410091:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d67e0. 00020000:00000010:0.0:1775767967.410092:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994209592f00. 00000100:00100000:1.0:1775767967.410093:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff99420788b100 x1862022294257536/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775767967.410096:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00008000:00000010:0.0:1775767967.410098:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff99422035b880. 00000100:00000001:2.0:1775767967.410103:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000010:0.0:1775767967.410104:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994207e3e150. 00000100:00000001:2.0:1775767967.410109:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1775767967.410110:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b2238. 00000100:00000001:1.0:1775767967.410110:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775767967.410112:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1775767967.410114:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0f600. 00000100:00000001:1.0:1775767967.410115:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767967.410116:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000008:00000010:0.0:1775767967.410120:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff99420906be40. 00000100:00000040:1.0:1775767967.410124:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e26cf977] to pc [ptlrpcd_rcv+-1] req@ffff99420788b100 x1862022294257536/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767967.410124:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00020000:00000010:0.0:1775767967.410125:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff9942103ccfc0. 00000100:00000001:2.0:1775767967.410127:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00008000:00000010:0.0:1775767967.410130:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076afa0. 00000100:00000001:2.0:1775767967.410132:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000080:00000010:0.0:1775767967.410135:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942107690f0. 00000100:00000001:2.0:1775767967.410136:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767967.410139:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000001:0.0:1775767967.410141:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107752095112 : -112965957456504 : ffff9942103bed88) 00000020:00001000:0.0:1775767967.410143:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff9942103bed80 00000100:00000001:1.0:1775767967.410144:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767967.410147:0:13471:0:(cl_object.c:151:cl_object_top()) ffff9942078a7f58 -> ffff9942078a7f58 00000100:00000040:2.0:1775767967.410147:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207888e00 x1862022294257408/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767967.410148:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000001:0.0:1775767967.410152:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1775767967.410175:0:13471:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:0.0:1775767967.410179:0:13471:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017b:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000100:00080000:1.0:1775767967.410180:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775767967.410184:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775767967.410187:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775767967.410188:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000001:0.0:1775767967.410189:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000100:00000001:2.0:1775767967.410191:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767967.410192:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00000001:1.0:1775767967.410192:0:8:0:(import.c:484:import_select_connection()) Process entered 02000000:00000001:2.0:1775767967.410195:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000080:00000001:0.0:1775767967.410196:0:13471:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767967.410198:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5219 02000000:00000001:2.0:1775767967.410198:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00020000:00000002:0.0:1775767967.410201:0:13471:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017b:0x0]io ffff9942077b2298 type 6 ignore/verify layout 1/0 02000000:00000001:2.0:1775767967.410204:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1775767967.410222:0:13471:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207c1f000) owner 0000000000000000/ffff994220080000 00000400:00000200:1.0:1775767967.410224:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:0.0:1775767967.410228:0:13471:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00000100:00100000:2.0:1775767967.410228:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207888e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294257408:192.168.202.156@tcp:38:kworker.0 00020000:00000001:0.0:1775767967.410233:0:13471:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00000400:00000200:1.0:1775767967.410234:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00020000:00000001:0.0:1775767967.410237:0:13471:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00000100:00000001:2.0:1775767967.410238:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00080000:1.0:1775767967.410239:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00020000:00000001:0.0:1775767967.410242:0:13471:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767967.410244:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:1.0:1775767967.410245:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00020000:00000001:0.0:1775767967.410247:0:13471:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775767967.410248:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1775767967.410248:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767967.410252:0:13471:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00000100:00000040:1.0:1775767967.410253:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000001:2.0:1775767967.410254:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00020000:00000001:0.0:1775767967.410255:0:13471:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767967.410257:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775767967.410258:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1775767967.410261:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 02000000:00000010:2.0:1775767967.410261:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1ec00. 00000020:00000001:1.0:1775767967.410262:0:8:0:(genops.c:811:class_conn2export()) Process entered 00020000:00000001:0.0:1775767967.410264:0:13471:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00000020:00000040:1.0:1775767967.410265:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 02000000:00000001:2.0:1775767967.410267:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1775767967.410268:0:13471:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207c1f000) owner 0000000000000000/ffff994220080000 00000020:00000001:1.0:1775767967.410268:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775767967.410271:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:0.0:1775767967.410274:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775767967.410274:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:0.0:1775767967.410279:0:13471:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:1.0:1775767967.410280:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000400:00000200:2.0:1775767967.410280:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:0.0:1775767967.410281:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00020000:00000001:0.0:1775767967.410284:0:13471:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00020000:00200000:0.0:1775767967.410287:0:13471:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00000040:1.0:1775767967.410287:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000010:2.0:1775767967.410288:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39688. 00000100:00000001:1.0:1775767967.410291:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00020000:00000002:0.0:1775767967.410292:0:13471:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00000400:00000010:2.0:1775767967.410295:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8fdd0. 00020000:00000001:0.0:1775767967.410296:0:13471:0:(lov_io.c:155:lov_sub_get()) Process entered 00000100:00000040:1.0:1775767967.410296:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00020000:00000001:0.0:1775767967.410298:0:13471:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000020:00000001:0.0:1775767967.410301:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00000200:2.0:1775767967.410301:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294257408, portal 10 00000100:00000001:1.0:1775767967.410302:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:2.0:1775767967.410305:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000020:00000010:0.0:1775767967.410306:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff9942103be000. 00000020:00000040:1.0:1775767967.410307:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00000001:2.0:1775767967.410308:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606121984 : -112966103429632 : ffff994207888e00) 00000020:00000010:0.0:1775767967.410310:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0fc00. 00000020:00000010:0.0:1775767967.410315:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210b0ec00. 00000100:00080000:1.0:1775767967.410316:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000020:00000010:0.0:1775767967.410320:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942078a3ea0. 00000100:00000040:2.0:1775767967.410320:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207888e00 x1862022294257408/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767992 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767967.410321:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767967.410327:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000008:00000010:0.0:1775767967.410328:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff9942086e4e18. 02000000:00000001:1.0:1775767967.410333:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000010:0.0:1775767967.410335:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994209592e40. 02000000:00000001:1.0:1775767967.410339:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767967.410341:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00008000:00000010:0.0:1775767967.410343:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994220358000. 00000100:00000001:1.0:1775767967.410343:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000080:00000010:0.0:1775767967.410349:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994207e3e3f0. 00000008:00100000:1.0:1775767967.410349:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000100:00000040:2.0:1775767967.410349:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000008:00000001:1.0:1775767967.410355:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775767967.410355:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8faa0. 00000080:00000010:0.0:1775767967.410356:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b0ff8. 00000100:00000001:1.0:1775767967.410361:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1775767967.410362:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294257408, offset 0 00000020:00000010:0.0:1775767967.410364:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0ea00. 00000100:00000010:1.0:1775767967.410369:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207889f80. 00000020:00000040:1.0:1775767967.410373:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00000008:00000010:0.0:1775767967.410374:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff99420906a6e0. 00000400:00000200:2.0:1775767967.410377:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00020000:00000010:0.0:1775767967.410380:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff9942103cc480. 02000000:00000001:1.0:1775767967.410380:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775767967.410383:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:0.0:1775767967.410385:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076adb0. 00000100:00000001:1.0:1775767967.410389:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:2.0:1775767967.410389:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000080:00000010:0.0:1775767967.410391:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff994210769870. 02000000:00000001:1.0:1775767967.410392:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000020:00000001:0.0:1775767967.410396:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107752091656 : -112965957459960 : ffff9942103be008) 00000400:00000200:2.0:1775767967.410396:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 02000000:00000001:1.0:1775767967.410397:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1775767967.410403:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff9942103be000 00000400:00000200:2.0:1775767967.410405:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000010:1.0:1775767967.410406:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543d800. 00000020:00000001:0.0:1775767967.410408:0:13471:0:(cl_object.c:151:cl_object_top()) ffff9942078a75f8 -> ffff9942078a7f58 00000400:00000200:2.0:1775767967.410412:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000020:00000001:0.0:1775767967.410413:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000100:00000001:1.0:1775767967.410413:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767967.410418:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000400:00000200:2.0:1775767967.410420:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000020:00000001:0.0:1775767967.410421:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00100000:1.0:1775767967.410424:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207889f80 x1862022294257664/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000001:0.0:1775767967.410425:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767967.410429:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00020000:00000001:0.0:1775767967.410431:0:13471:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767967.410437:0:13471:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107752153128 : -112965957398488 : ffff9942103cd028) 00000400:00000200:2.0:1775767967.410439:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225184:225184:255:4294967295] 192.168.202.156@tcp LPNI seq info [225180:225180:7:4294967295] 00000020:00000001:0.0:1775767967.410444:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000020:00000001:0.0:1775767967.410447:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767967.410451:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00020000:00000001:0.0:1775767967.410452:0:13471:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767967.410455:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:0.0:1775767967.410457:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767967.410457:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767967.410463:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00020000:00000001:0.0:1775767967.410466:0:13471:0:(lov_io.c:1108:lov_io_lock()) Process entered 00000100:00000040:1.0:1775767967.410468:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ed2274f4] to pc [ptlrpcd_rcv+-1] req@ffff994207889f80 x1862022294257664/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:0.0:1775767967.410471:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:0.0:1775767967.410475:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:0.0:1775767967.410479:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000400:00000200:2.0:1775767967.410480:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000020:00000001:0.0:1775767967.410483:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767967.410488:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767967.410493:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767967.410497:0:13471:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1775767967.410497:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00000001:1.0:1775767967.410500:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767967.410503:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:0.0:1775767967.410506:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1775767967.410506:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3600. 00000100:00080000:1.0:1775767967.410507:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000001:0.0:1775767967.410512:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767967.410514:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775767967.410518:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000800:00000200:2.0:1775767967.410518:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00080000:1.0:1775767967.410521:0:8:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:2.1:1775767967.410527:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:1.0:1775767967.410529:0:8:0:(import.c:484:import_select_connection()) Process entered 00000020:00000001:0.0:1775767967.410532:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00000100:00000001:2.0:1775767967.410534:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767967.410536:0:13471:0:(lov_io.c:1114:lov_io_start()) Process entered 00000100:00080000:1.0:1775767967.410536:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5219 00020000:00000001:0.0:1775767967.410540:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000040:2.0:1775767967.410542:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000020:00000001:0.0:1775767967.410543:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00000008:00000001:0.0:1775767967.410545:0:13471:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000400:00000200:1.0:1775767967.410548:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00000001:0.0:1775767967.410549:0:13471:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000400:00000200:2.0:1775767967.410554:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00000001:0.0:1775767967.410555:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000400:00000200:1.0:1775767967.410557:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000008:00000020:0.0:1775767967.410560:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000100:00000001:2.0:1775767967.410560:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299892243 00000100:00080000:1.0:1775767967.410561:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000008:00000001:0.0:1775767967.410564:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1775767967.410570:0:13471:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000100:00000040:1.0:1775767967.410570:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767967.410572:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299893565 00000008:00000001:0.0:1775767967.410574:0:13471:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000100:00000001:1.0:1775767967.410576:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000008:00000001:0.0:1775767967.410579:0:13471:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107884688256 : -112965824863360 : ffff994218232380) 00000100:00000001:2.0:1775767967.410581:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299892243 00000100:00000040:1.0:1775767967.410582:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000008:00000020:0.0:1775767967.410585:0:13471:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994218232380 ready 0|-|- wr 1|-|+ rd 0|- 9 in flight 00000100:00000001:2.0:1775767967.410588:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299893565 00000100:00000001:1.0:1775767967.410590:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:2.0:1775767967.410595:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1775767967.410596:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000020:00000001:1.0:1775767967.410597:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000008:00000020:0.0:1775767967.410601:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000100:00000001:2.0:1775767967.410601:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1775767967.410602:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000008:00000001:0.0:1775767967.410605:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1775767967.410607:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000008:00000001:0.0:1775767967.410608:0:13471:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000100:00000001:2.0:1775767967.410608:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000008:00000020:0.0:1775767967.410612:0:13471:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994210674fc0 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000020:00000040:1.0:1775767967.410612:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000100:00000001:2.0:1775767967.410613:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1775767967.410618:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000001:2.0:1775767967.410621:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000008:00000001:0.0:1775767967.410622:0:13471:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1775767967.410623:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000001:2.0:1775767967.410625:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000008:00000001:0.0:1775767967.410627:0:13471:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767967.410631:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1775767967.410632:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767967.410632:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767967.410635:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00020000:00000001:0.0:1775767967.410638:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767967.410639:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775767967.410639:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00020000:00000001:0.0:1775767967.410645:0:13471:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767967.410646:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000040:2.0:1775767967.410649:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff99420788b100 x1862022294257536/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775767967.410650:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767967.410651:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775767967.410655:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00020000:00000001:0.0:1775767967.410659:0:13471:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00000020:00000040:1.0:1775767967.410659:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00020000:00000001:0.0:1775767967.410663:0:13471:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000020:00000001:0.0:1775767967.410667:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00000100:00080000:1.0:1775767967.410669:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000020:00000001:0.0:1775767967.410670:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00000100:00000001:2.0:1775767967.410673:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00020000:00000001:0.0:1775767967.410674:0:13471:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767967.410676:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767967.410677:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767967.410678:0:13471:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000020:00000001:0.0:1775767967.410682:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 02000000:00000001:1.0:1775767967.410683:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000020:00000001:0.0:1775767967.410686:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 02000000:00000001:1.0:1775767967.410689:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1775767967.410690:0:13471:0:(lov_io.c:1208:lov_io_unlock()) Process entered 02000000:00000001:2.0:1775767967.410690:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00020000:00000001:0.0:1775767967.410693:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 02000000:00000001:1.0:1775767967.410694:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767967.410696:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 02000000:00000001:2.0:1775767967.410696:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775767967.410698:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000020:00000001:0.0:1775767967.410700:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:0.0:1775767967.410703:0:13471:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:1.0:1775767967.410703:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 02000000:00000001:2.0:1775767967.410703:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767967.410709:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1775767967.410711:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767967.410713:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff99420788b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294257536:192.168.202.156@tcp:38:kworker.0 00020000:00000001:0.0:1775767967.410714:0:13471:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000100:00000001:1.0:1775767967.410716:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767967.410718:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:0.0:1775767967.410722:0:13471:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000100:00000010:1.0:1775767967.410724:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207888700. 00000020:00000001:0.0:1775767967.410725:0:13471:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000100:00000001:2.0:1775767967.410725:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000001:0.0:1775767967.410729:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000040:1.0:1775767967.410730:0:8:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:2.0:1775767967.410730:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00020000:00000001:0.0:1775767967.410733:0:13471:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 02000000:00000001:2.0:1775767967.410735:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767967.410736:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 02000000:00000001:1.0:1775767967.410737:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000020:00000001:0.0:1775767967.410740:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 02000000:00000001:2.0:1775767967.410740:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1775767967.410741:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767967.410744:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000100:00000001:1.0:1775767967.410745:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00020000:00000001:0.0:1775767967.410746:0:13471:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775767967.410747:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f000. 02000000:00000001:1.0:1775767967.410748:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00020000:00000001:0.0:1775767967.410753:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767967.410754:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767967.410754:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767967.410757:0:13471:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000020:00000001:0.0:1775767967.410761:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 02000000:00000010:1.0:1775767967.410764:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543dc00. 00000020:00000001:0.0:1775767967.410765:0:13471:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767967.410765:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:0.0:1775767967.410771:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00000100:00000001:1.0:1775767967.410772:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767967.410773:0:13471:0:(lov_io.c:675:lov_io_fini()) Process entered 00000400:00000010:2.0:1775767967.410773:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39948. 00020000:00000001:0.0:1775767967.410777:0:13471:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:0.0:1775767967.410779:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00000400:00000010:2.0:1775767967.410781:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f990. 00000020:00000001:0.0:1775767967.410782:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000100:00100000:1.0:1775767967.410783:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207888700 x1862022294257792/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00001000:0.0:1775767967.410787:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff9942103be000 00000100:00000200:2.0:1775767967.410788:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294257536, portal 10 00000020:00000010:0.0:1775767967.410792:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210b0ec00. 00000100:00000001:2.0:1775767967.410795:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000020:00000010:0.0:1775767967.410799:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942078a3ea0. 00000100:00000001:2.0:1775767967.410799:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606130944 : -112966103420672 : ffff99420788b100) 00000008:00000010:0.0:1775767967.410806:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff9942086e4e18. 00000100:00000001:1.0:1775767967.410807:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00020000:00000010:0.0:1775767967.410812:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994209592e40. 00000100:00000001:1.0:1775767967.410813:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767967.410813:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff99420788b100 x1862022294257536/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767992 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00008000:00000010:0.0:1775767967.410818:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994220358000. 00000100:00000040:1.0:1775767967.410824:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009f2ba301] to pc [ptlrpcd_rcv+-1] req@ffff994207888700 x1862022294257792/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000010:0.0:1775767967.410825:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994207e3e3f0. 00000080:00000010:0.0:1775767967.410832:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b0ff8. 00000020:00000010:0.0:1775767967.410839:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0fc00. 00000100:00000001:2.0:1775767967.410842:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00000010:0.0:1775767967.410845:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff99420906a6e0. 00000100:00000040:2.0:1775767967.410851:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00020000:00000010:0.0:1775767967.410852:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff9942103cc480. 00000100:00000001:1.0:1775767967.410856:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:0.0:1775767967.410857:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076adb0. 00000400:00000010:2.0:1775767967.410857:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8fe58. 00000080:00000010:0.0:1775767967.410863:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff994210769870. 00000100:00000040:1.0:1775767967.410865:0:8:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5229) 00000100:00000200:2.0:1775767967.410865:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294257536, offset 0 00000020:00000010:0.0:1775767967.410868:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0ea00. 00000020:00000010:0.0:1775767967.410874:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff9942103be000. 00020000:00000001:0.0:1775767967.410879:0:13471:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00000400:00000200:2.0:1775767967.410879:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00020000:00000001:0.0:1775767967.410884:0:13471:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:0.0:1775767967.410887:0:13471:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000400:00000200:2.0:1775767967.410888:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000080:00200000:0.0:1775767967.410891:0:13471:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000400:00000200:2.0:1775767967.410896:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000080:00000001:0.0:1775767967.410897:0:13471:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1775767967.410901:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000400:00000200:2.0:1775767967.410904:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00001000:0.0:1775767967.410905:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff9942103bed80 00000020:00000010:0.0:1775767967.410909:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210b0f200. 00000400:00000200:2.0:1775767967.410910:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000020:00000010:0.0:1775767967.410915:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942078a3d80. 00000400:00000200:2.0:1775767967.410915:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767967.410921:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000008:00000010:0.0:1775767967.410922:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff9942086e6468. 00020000:00000010:0.0:1775767967.410928:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994209592f00. 00000400:00000200:2.0:1775767967.410928:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225185:225185:254:4294967295] 192.168.202.156@tcp LPNI seq info [225181:225181:6:4294967295] 00008000:00000010:0.0:1775767967.410934:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff99422035b880. 00000400:00000200:2.0:1775767967.410939:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000080:00000010:0.0:1775767967.410940:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994207e3e150. 00000080:00000010:0.0:1775767967.410945:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b2238. 00000020:00000010:0.0:1775767967.410951:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0fa00. 00000400:00000200:2.0:1775767967.410955:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000008:00000010:0.0:1775767967.410957:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff99420906be40. 00020000:00000010:0.0:1775767967.410963:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff9942103ccfc0. 00000800:00000200:2.0:1775767967.410965:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00008000:00000010:0.0:1775767967.410970:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076afa0. 00000800:00000010:2.0:1775767967.410972:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3800. 00000080:00000010:0.0:1775767967.410976:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942107690f0. 00000800:00000200:2.0:1775767967.410981:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000010:0.0:1775767967.410982:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0f600. 00000020:00000010:0.0:1775767967.410987:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff9942103bed80. 00000800:00000200:2.1:1775767967.410990:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000080:00000001:0.0:1775767967.410991:0:13471:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1775767967.410997:0:13471:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767967.410997:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767967.411004:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000080:00000001:0.0:1775767967.411005:0:13471:0:(rw.c:1481:ll_writepages()) Process entered 00000080:00000001:0.0:1775767967.411008:0:13471:0:(file.c:5250:cl_sync_file_range()) Process entered 00000400:00000200:2.0:1775767967.411012:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:0.0:1775767967.411013:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00000001:2.0:1775767967.411016:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299892235 00000100:00000001:2.0:1775767967.411021:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299893565 00000020:00000010:0.0:1775767967.411022:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff9942103bed80. 00000100:00000001:2.0:1775767967.411025:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299892235 00000020:00000010:0.0:1775767967.411029:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0f600. 00000100:00000001:2.0:1775767967.411029:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299893565 00000100:00000001:2.0:1775767967.411035:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1775767967.411038:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210b0fa00. 00000100:00000001:2.0:1775767967.411039:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1775767967.411044:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942078a3d80. 00000100:00000001:2.0:1775767967.411045:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000008:00000010:0.0:1775767967.411051:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff9942086e6468. 00000100:00000040:2.0:1775767967.411052:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207889f80 x1862022294257664/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000010:0.0:1775767967.411057:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994209592f00. 00008000:00000010:0.0:1775767967.411063:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff99422035b880. 00000100:00000001:2.0:1775767967.411068:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000080:00000010:0.0:1775767967.411069:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994207e3e150. 00000100:00000001:2.0:1775767967.411071:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1775767967.411075:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b2238. 02000000:00000001:2.0:1775767967.411076:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767967.411080:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1775767967.411082:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0f200. 02000000:00000001:2.0:1775767967.411085:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:0.0:1775767967.411087:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff99420906be40. 00000100:00100000:2.0:1775767967.411091:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207889f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294257664:192.168.202.156@tcp:8:kworker.0 00020000:00000010:0.0:1775767967.411093:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff9942103ccfc0. 00000100:00000001:2.0:1775767967.411099:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00008000:00000010:0.0:1775767967.411100:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076afa0. 02000000:00000001:2.0:1775767967.411103:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000080:00000010:0.0:1775767967.411105:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942107690f0. 02000000:00000001:2.0:1775767967.411106:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767967.411109:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107752095112 : -112965957456504 : ffff9942103bed88) 02000000:00000001:2.0:1775767967.411110:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00001000:0.0:1775767967.411114:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff9942103bed80 02000000:00000010:2.0:1775767967.411116:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1e800. 00000020:00000001:0.0:1775767967.411119:0:13471:0:(cl_object.c:151:cl_object_top()) ffff99420791a850 -> ffff99420791a850 02000000:00000001:2.0:1775767967.411120:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767967.411123:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1775767967.411127:0:13471:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000400:00000200:2.0:1775767967.411127:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000080:00200000:0.0:1775767967.411132:0:13471:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017f:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000400:00000010:2.0:1775767967.411133:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39528. 00000020:00000001:0.0:1775767967.411140:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000400:00000010:2.0:1775767967.411140:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f198. 00000020:00000001:0.0:1775767967.411142:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1775767967.411145:0:13471:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1775767967.411146:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294257664, portal 4 00020000:00000002:0.0:1775767967.411148:0:13471:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017f:0x0]io ffff9942077b2298 type 6 ignore/verify layout 1/0 00000100:00000001:2.0:1775767967.411149:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767967.411152:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606126464 : -112966103425152 : ffff994207889f80) 00000100:00000040:2.0:1775767967.411185:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207889f80 x1862022294257664/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767992 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000002:0.0:1775767967.411187:0:13471:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207dfbd90) owner 0000000000000000/ffff994220080000 00020000:00000001:0.0:1775767967.411194:0:13471:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1775767967.411197:0:13471:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00000100:00000001:2.0:1775767967.411199:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00020000:00000001:0.0:1775767967.411200:0:13471:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00020000:00000001:0.0:1775767967.411204:0:13471:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767967.411205:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00020000:00000001:0.0:1775767967.411222:0:13471:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1775767967.411225:0:13471:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1775767967.411227:0:13471:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775767967.411228:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f880. 00000020:00000001:0.0:1775767967.411230:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1775767967.411233:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00000200:2.0:1775767967.411235:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294257664, offset 0 00020000:00000001:0.0:1775767967.411236:0:13471:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00020000:00000002:0.0:1775767967.411239:0:13471:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207dfbd90) owner 0000000000000000/ffff994220080000 00000020:00000001:0.0:1775767967.411243:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767967.411245:0:13471:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:0.0:1775767967.411247:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00020000:00000001:0.0:1775767967.411251:0:13471:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00000400:00000200:2.0:1775767967.411252:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00020000:00200000:0.0:1775767967.411255:0:13471:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00000002:0.0:1775767967.411259:0:13471:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00000400:00000200:2.0:1775767967.411259:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:0.0:1775767967.411265:0:13471:0:(lov_io.c:155:lov_sub_get()) Process entered 00000400:00000200:2.0:1775767967.411268:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00020000:00000001:0.0:1775767967.411269:0:13471:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000020:00000001:0.0:1775767967.411272:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000400:00000200:2.0:1775767967.411274:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000010:0.0:1775767967.411277:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff9942103be000. 00000400:00000200:2.0:1775767967.411278:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000020:00000010:0.0:1775767967.411284:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0ea00. 00000400:00000200:2.0:1775767967.411284:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000020:00000010:0.0:1775767967.411290:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210b0fc00. 00000400:00000200:2.0:1775767967.411291:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000020:00000010:0.0:1775767967.411297:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942078a3ea0. 00000400:00000200:2.0:1775767967.411297:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225186:225186:253:4294967295] 192.168.202.156@tcp LPNI seq info [225182:225182:5:4294967295] 00000400:00000200:2.0:1775767967.411304:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00000010:0.0:1775767967.411305:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff9942086e4e18. 00020000:00000010:0.0:1775767967.411309:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994209592e40. 00000400:00000200:2.0:1775767967.411315:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00008000:00000010:0.0:1775767967.411320:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994220358000. 00000800:00000200:2.0:1775767967.411321:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000080:00000010:0.0:1775767967.411325:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994207e3e3f0. 00000800:00000010:2.0:1775767967.411328:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3500. 00000080:00000010:0.0:1775767967.411330:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b0ff8. 00000020:00000010:0.0:1775767967.411336:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0ec00. 00000800:00000200:2.0:1775767967.411336:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000008:00000010:0.0:1775767967.411342:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff99420906a6e0. 00000800:00000200:2.1:1775767967.411346:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00020000:00000010:0.0:1775767967.411347:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff9942103cc480. 00000100:00000001:2.0:1775767967.411351:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:0.0:1775767967.411352:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076adb0. 00000080:00000010:0.0:1775767967.411358:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff994210769870. 00000100:00000040:2.0:1775767967.411359:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000020:00000001:0.0:1775767967.411363:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107752091656 : -112965957459960 : ffff9942103be008) 00000400:00000200:2.0:1775767967.411368:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00001000:0.0:1775767967.411369:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff9942103be000 00000100:00000001:2.0:1775767967.411372:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299892259 00000020:00000001:0.0:1775767967.411374:0:13471:0:(cl_object.c:151:cl_object_top()) ffff99420791a210 -> ffff99420791a850 00000020:00000001:0.0:1775767967.411379:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000100:00000001:2.0:1775767967.411379:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299893565 00000020:00000001:0.0:1775767967.411381:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1775767967.411384:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00000001:2.0:1775767967.411385:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299892259 00000020:00000001:0.0:1775767967.411386:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767967.411392:0:13471:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767967.411392:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299893565 00020000:00000001:0.0:1775767967.411397:0:13471:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107752153128 : -112965957398488 : ffff9942103cd028) 00000100:00000001:2.0:1775767967.411398:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1775767967.411402:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000100:00000001:2.0:1775767967.411403:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767967.411405:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767967.411409:0:13471:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767967.411410:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767967.411412:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767967.411414:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1775767967.411417:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00020000:00000001:0.0:1775767967.411420:0:13471:0:(lov_io.c:1108:lov_io_lock()) Process entered 00000100:00000001:2.0:1775767967.411421:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00020000:00000001:0.0:1775767967.411422:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:0.0:1775767967.411425:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00000100:00000001:2.0:1775767967.411426:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000020:00000001:0.0:1775767967.411427:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:0.0:1775767967.411432:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767967.411432:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1775767967.411436:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767967.411437:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00020000:00000001:0.0:1775767967.411440:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767967.411441:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00020000:00000001:0.0:1775767967.411444:0:13471:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767967.411448:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000100:00000040:2.0:1775767967.411449:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207888700 x1862022294257792/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775767967.411450:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767967.411455:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767967.411459:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00020000:00000001:0.0:1775767967.411461:0:13471:0:(lov_io.c:1114:lov_io_start()) Process entered 00020000:00000001:0.0:1775767967.411463:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:0.0:1775767967.411466:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00000008:00000001:0.0:1775767967.411468:0:13471:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000100:00000001:2.0:1775767967.411470:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000008:00000001:0.0:1775767967.411471:0:13471:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000100:00000001:2.0:1775767967.411474:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1775767967.411477:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 02000000:00000001:2.0:1775767967.411479:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000020:0.0:1775767967.411480:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:0.0:1775767967.411484:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1775767967.411486:0:13471:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000008:00000001:0.0:1775767967.411487:0:13471:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000008:00000001:0.0:1775767967.411489:0:13471:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107754938304 : -112965954613312 : ffff994210674fc0) 00000008:00000020:0.0:1775767967.411492:0:13471:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994210674fc0 ready 0|-|- wr 1|-|+ rd 0|- 9 in flight 02000000:00000001:2.0:1775767967.411494:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1775767967.411495:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:0.0:1775767967.411498:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 02000000:00000001:2.0:1775767967.411499:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1775767967.411501:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767967.411506:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207888700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294257792:192.168.202.156@tcp:8:kworker.0 00000008:00000001:0.0:1775767967.411510:0:13471:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000100:00000001:2.0:1775767967.411511:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000008:00000020:0.0:1775767967.411514:0:13471:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994218232380 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 02000000:00000001:2.0:1775767967.411515:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767967.411519:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1775767967.411521:0:13471:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1775767967.411523:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:0.0:1775767967.411524:0:13471:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767967.411527:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767967.411530:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775767967.411530:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1d400. 00020000:00000001:0.0:1775767967.411532:0:13471:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767967.411534:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767967.411536:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767967.411539:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00020000:00000001:0.0:1775767967.411542:0:13471:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00000400:00000200:2.0:1775767967.411543:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:0.0:1775767967.411544:0:13471:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000020:00000001:0.0:1775767967.411547:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00000400:00000010:2.0:1775767967.411549:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39aa8. 00000020:00000001:0.0:1775767967.411550:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00020000:00000001:0.0:1775767967.411552:0:13471:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767967.411555:0:13471:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000020:00000001:0.0:1775767967.411557:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00000400:00000010:2.0:1775767967.411557:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f5d8. 00000020:00000001:0.0:1775767967.411558:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00020000:00000001:0.0:1775767967.411561:0:13471:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00020000:00000001:0.0:1775767967.411563:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000200:2.0:1775767967.411564:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294257792, portal 4 00000020:00000001:0.0:1775767967.411565:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000020:00000001:0.0:1775767967.411567:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000100:00000001:2.0:1775767967.411569:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00020000:00000001:0.0:1775767967.411570:0:13471:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767967.411573:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606120192 : -112966103431424 : ffff994207888700) 00020000:00000001:0.0:1775767967.411574:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767967.411579:0:13471:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000020:00000001:0.0:1775767967.411581:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:0.0:1775767967.411584:0:13471:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000100:00000040:2.0:1775767967.411585:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207888700 x1862022294257792/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767992 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775767967.411586:0:13471:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000001:0.0:1775767967.411588:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:0.0:1775767967.411590:0:13471:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00020000:00000001:0.0:1775767967.411593:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:0.0:1775767967.411596:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000001:0.0:1775767967.411598:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00020000:00000001:0.0:1775767967.411600:0:13471:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767967.411602:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00020000:00000001:0.0:1775767967.411603:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775767967.411607:0:13471:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000100:00000040:2.0:1775767967.411609:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000020:00000001:0.0:1775767967.411610:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000020:00000001:0.0:1775767967.411613:0:13471:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767967.411615:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00000400:00000010:2.0:1775767967.411615:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00020000:00000001:0.0:1775767967.411618:0:13471:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:0.0:1775767967.411620:0:13471:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000100:00000200:2.0:1775767967.411621:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294257792, offset 0 00000020:00000001:0.0:1775767967.411622:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:0.0:1775767967.411624:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1775767967.411626:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff9942103be000 00000020:00000010:0.0:1775767967.411630:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210b0fc00. 00000400:00000200:2.0:1775767967.411634:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000020:00000010:0.0:1775767967.411636:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942078a3ea0. 00000008:00000010:0.0:1775767967.411640:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff9942086e4e18. 00000400:00000200:2.0:1775767967.411641:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000010:0.0:1775767967.411647:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994209592e40. 00000400:00000200:2.0:1775767967.411649:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00008000:00000010:0.0:1775767967.411652:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994220358000. 00000400:00000200:2.0:1775767967.411655:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000080:00000010:0.0:1775767967.411657:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994207e3e3f0. 00000400:00000200:2.0:1775767967.411660:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000080:00000010:0.0:1775767967.411662:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b0ff8. 00000400:00000200:2.0:1775767967.411666:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000020:00000010:0.0:1775767967.411668:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0ea00. 00000008:00000010:0.0:1775767967.411672:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff99420906a6e0. 00000400:00000200:2.0:1775767967.411672:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00020000:00000010:0.0:1775767967.411677:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff9942103cc480. 00000400:00000200:2.0:1775767967.411681:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225187:225187:252:4294967295] 192.168.202.156@tcp LPNI seq info [225183:225183:4:4294967295] 00008000:00000010:0.0:1775767967.411683:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076adb0. 00000080:00000010:0.0:1775767967.411689:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff994210769870. 00000400:00000200:2.0:1775767967.411693:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000010:0.0:1775767967.411695:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0ec00. 00000020:00000010:0.0:1775767967.411699:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff9942103be000. 00020000:00000001:0.0:1775767967.411705:0:13471:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:0.0:1775767967.411709:0:13471:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000400:00000200:2.0:1775767967.411710:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000080:00000001:0.0:1775767967.411711:0:13471:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000080:00200000:0.0:1775767967.411715:0:13471:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017f:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000800:00000200:2.0:1775767967.411720:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000080:00000001:0.0:1775767967.411722:0:13471:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1775767967.411725:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000800:00000010:2.0:1775767967.411726:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3a00. 00000020:00001000:0.0:1775767967.411727:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff9942103bed80 00000020:00000010:0.0:1775767967.411732:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210b0fa00. 00000020:00000010:0.0:1775767967.411737:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942078a3d80. 00000800:00000200:2.0:1775767967.411737:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000008:00000010:0.0:1775767967.411744:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff9942086e6468. 00000800:00000200:2.1:1775767967.411748:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00020000:00000010:0.0:1775767967.411750:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994209592f00. 00008000:00000010:0.0:1775767967.411754:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff99422035b880. 00000100:00000001:2.0:1775767967.411755:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1775767967.411758:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994207e3e150. 00000100:00000040:2.0:1775767967.411759:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000080:00000010:0.0:1775767967.411762:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b2238. 00000020:00000010:0.0:1775767967.411767:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0f600. 00000400:00000200:2.0:1775767967.411767:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767967.411771:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299892262 00000008:00000010:0.0:1775767967.411772:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff99420906be40. 00020000:00000010:0.0:1775767967.411778:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff9942103ccfc0. 00000100:00000001:2.0:1775767967.411778:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299893565 00008000:00000010:0.0:1775767967.411783:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076afa0. 00000100:00000001:2.0:1775767967.411783:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299892262 00000080:00000010:0.0:1775767967.411787:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942107690f0. 00000100:00000001:2.0:1775767967.411788:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299893565 00000020:00000010:0.0:1775767967.411793:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0f200. 00000100:00000001:2.0:1775767967.411793:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1775767967.411797:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff9942103bed80. 00000100:00000001:2.0:1775767967.411798:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1775767967.411802:0:13471:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767967.411804:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1775767967.411806:0:13471:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767967.411808:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767967.411814:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767967.411816:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1775767967.411818:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775767967.411820:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767967.411823:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767967.411825:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767967.473254:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767967.473259:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767967.473264:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767967.473265:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767967.473268:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767967.473269:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767967.473271:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767967.473272:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767967.473275:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767967.473276:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767967.473291:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767967.473293:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767967.473295:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767967.537251:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767967.537258:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767967.537259:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767967.537264:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767967.537267:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767967.537270:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767967.537272:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767967.537274:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767967.537276:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767968.433239:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767968.433246:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767968.433249:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767968.433253:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775767968.433257:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767968.433260:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767968.433263:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767968.433265:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767968.433268:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767968.433292:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767968.433297:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767968.433299:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767968.433303:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767968.433307:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767968.433309:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767968.433310:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767968.433319:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767968.433319:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767968.433323:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767968.433324:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767968.433334:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767968.433335:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767968.433345:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767968.433349:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767968.433354:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767968.433357:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767968.433360:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767968.497303:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767968.497306:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767968.497309:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767968.497310:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767968.497312:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767968.497313:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767968.497316:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767968.497317:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767968.497319:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767968.497320:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767968.497332:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767968.497334:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767968.497336:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767968.561283:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767968.561289:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767968.561291:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767968.561295:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767968.561300:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767968.561304:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767968.561307:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767968.561309:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767968.561313:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767968.753288:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299893565, at 4299893568 00000100:00000040:0.0:1775767968.753289:0:14953:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4299893565, at 4299893568 00000100:00000010:0.0:1775767968.753306:0:14953:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d6540. 00000100:00000010:1.0:1775767968.753309:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d67e0. 00000100:00000001:1.0:1775767969.457316:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767969.457316:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767969.457321:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767969.457322:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767969.457324:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767969.457326:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767969.457332:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775767969.457332:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775767969.457337:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767969.457338:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767969.457342:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767969.457342:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767969.457347:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767969.457350:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767969.457352:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767969.457356:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767969.457359:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767969.457362:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767969.457406:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767969.457411:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767969.457412:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767969.457417:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767969.457419:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767969.457423:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767969.457425:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767969.457427:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767969.457430:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767969.521251:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767969.521256:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767969.521260:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767969.521261:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767969.521263:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767969.521264:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767969.521266:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767969.521267:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767969.521269:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767969.521270:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767969.521295:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767969.521298:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767969.521299:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767969.585243:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767969.585248:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767969.585250:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767969.585254:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767969.585258:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767969.585261:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767969.585263:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767969.585264:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767969.585266:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767970.481261:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767970.481277:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767970.481280:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767970.481286:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775767970.481290:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767970.481295:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767970.481298:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767970.481300:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767970.481303:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767970.481765:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767970.481767:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767970.481768:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767970.481770:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767970.481772:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767970.481774:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767970.481776:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767970.481778:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767970.481779:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767970.481806:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767970.481810:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767970.481812:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767970.481815:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775767970.481817:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767970.481819:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767970.481821:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767970.481823:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767970.481825:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:2.0:1775767970.483347:0:2390:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.156:988 00000400:00000100:2.0:1775767970.483407:0:2390:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.156@tcp at host 192.168.202.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:2.0:1775767970.483421:0:2390:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767970.483427:0:2390:0:(router.c:1750:lnet_notify()) 192.168.202.56@tcp notifying 192.168.202.156@tcp: down 00000400:00000200:2.0:1775767970.483441:0:2390:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767970.483446:0:2390:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:2.0:1775767970.483454:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000800:00000010:2.0:1775767970.483460:0:2390:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3300. 00000400:00000200:2.0:1775767970.483468:0:2390:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775767970.483477:0:2390:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775767970.483489:0:2390:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775767970.483492:0:2390:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775767970.483498:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767970.483513:0:2390:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff994207889880 x1862022294257280/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767992 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767970.483542:0:2390:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767970.483547:0:2390:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207889880 x1862022294257280/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767992 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767970.483557:0:2390:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767970.483561:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1775767970.483575:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000800:00000010:2.0:1775767970.483578:0:2390:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3600. 00000400:00000200:2.0:1775767970.483583:0:2390:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775767970.483588:0:2390:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775767970.483592:0:2390:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8faa0 00000400:00000010:2.0:1775767970.483595:0:2390:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8faa0. 00000100:00000001:2.0:1775767970.483599:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767970.483604:0:2390:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff994207888e00 x1862022294257408/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767992 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767970.483616:0:2390:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767970.483620:0:2390:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207888e00 x1862022294257408/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767992 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767970.483627:0:2390:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767970.483630:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767970.483637:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000100:2.0:1775767970.483638:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000800:00000010:2.0:1775767970.483643:0:2390:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3800. 00000100:00000001:3.0:1775767970.483644:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000400:00000200:2.0:1775767970.483648:0:2390:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000100:00000001:3.0:1775767970.483650:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000400:00000200:2.0:1775767970.483656:0:2390:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775767970.483659:0:2390:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8fe58 00000400:00000010:2.0:1775767970.483663:0:2390:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8fe58. 00000100:00100000:3.0:1775767970.483663:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767967/real 1775767970] req@ffff994207888e00 x1862022294257408/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767970.483668:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767970.483677:0:2390:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff99420788b100 x1862022294257536/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767992 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767970.483685:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767970.483689:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1775767970.483691:0:2390:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000400:00000010:3.0:1775767970.483694:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39688. 00000100:00000040:2.0:1775767970.483696:0:2390:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff99420788b100 x1862022294257536/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767992 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1775767970.483700:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8fdd0 00000400:00000010:3.0:1775767970.483704:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8fdd0. 00000100:00000001:3.0:1775767970.483710:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1775767970.483714:0:2390:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767970.483717:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000200:3.0:1775767970.483719:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207888e00 x1862022294257408/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1775767970.483729:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000800:00000010:2.0:1775767970.483736:0:2390:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3500. 00000100:00000200:3.0:1775767970.483740:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207888e00 x1862022294257408/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1775767970.483748:0:2390:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000100:00000001:3.0:1775767970.483750:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767970.483754:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1775767970.483758:0:2390:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000100:00000001:3.0:1775767970.483760:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000400:00000200:2.0:1775767970.483763:0:2390:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f880 00000100:00000001:3.0:1775767970.483765:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:2.0:1775767970.483768:0:2390:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f880. 00000100:00100000:3.0:1775767970.483773:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207888e00 x1862022294257408/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767970.483775:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767970.483784:0:2390:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff994207889f80 x1862022294257664/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767992 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767970.483786:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767970.483798:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207888e00 x1862022294257408/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767992 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767970.483800:0:2390:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767970.483808:0:2390:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207889f80 x1862022294257664/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767992 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767970.483815:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767970.483821:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767970.483824:0:2390:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767970.483825:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767970.483829:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767970.483829:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767970.483834:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000800:00000100:2.0:1775767970.483839:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000100:00000001:3.0:1775767970.483841:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000800:00000010:2.0:1775767970.483843:0:2390:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3a00. 00000100:00080000:3.0:1775767970.483848:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000400:00000200:2.0:1775767970.483849:0:2390:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775767970.483857:0:2390:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775767970.483863:0:2390:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775767970.483865:0:2390:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775767970.483867:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767970.483872:0:2390:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff994207888700 x1862022294257792/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767992 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767970.483886:0:2390:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767970.483888:0:2390:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207888700 x1862022294257792/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767992 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767970.483895:0:2390:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767970.483897:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:3.0:1775767970.483947:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767970.483951:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767970.483958:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207888e00 x1862022294257408/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767992 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767970.483967:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207888e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294257408:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767970.483973:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767970.483977:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767967/real 1775767970] req@ffff994207889880 x1862022294257280/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767970.483986:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767970.483988:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767970.483990:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39c60. 00000400:00000200:3.0:1775767970.483999:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f220 00000400:00000010:3.0:1775767970.484002:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f220. 00000100:00000001:3.0:1775767970.484005:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767970.484009:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207889880 x1862022294257280/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767970.484019:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207889880 x1862022294257280/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767970.484026:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767970.484028:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767970.484030:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767970.484031:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767970.484052:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207889880 x1862022294257280/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767970.484060:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767970.484065:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207889880 x1862022294257280/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767992 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767970.484074:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767970.484077:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767970.484078:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767970.484080:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767970.484083:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767970.484085:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767970.484087:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767970.484091:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767970.484094:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767970.484098:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207889880 x1862022294257280/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767992 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767970.484107:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207889880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294257280:192.168.202.156@tcp:250:kworker.0 00000100:00000001:3.0:1775767970.484112:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767970.484114:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767970.484118:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207888e00 x1862022294257408/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767992 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767970.484126:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767970.484129:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767970.484134:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1ec00. 02000000:00000001:3.0:1775767970.484139:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767970.484142:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767970.484144:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:3.0:1775767970.484147:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767970.484149:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543d000. 02000000:00000001:3.0:1775767970.484196:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767970.484199:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767970.484202:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207888e00. 00000100:00000001:3.0:1775767970.484207:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767970.484209:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767970.484211:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767970.484215:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207889880 x1862022294257280/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767992 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767970.484222:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767970.484224:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767970.484226:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1c800. 02000000:00000001:3.0:1775767970.484229:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767970.484230:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767970.484232:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:3.0:1775767970.484235:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767970.484236:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543fc00. 02000000:00000001:3.0:1775767970.484239:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767970.484241:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767970.484242:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207889880. 00000100:00000001:3.0:1775767970.484244:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767970.484246:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767970.484248:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767970.484251:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767970.484252:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767970.484254:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767970.484258:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767967/real 1775767970] req@ffff994207888700 x1862022294257792/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767970.484268:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767970.484269:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767970.484271:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39aa8. 00000400:00000200:3.0:1775767970.484274:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f5d8 00000400:00000010:3.0:1775767970.484276:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f5d8. 00000100:00000001:3.0:1775767970.484279:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767970.484282:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207888700 x1862022294257792/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767970.484291:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207888700 x1862022294257792/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767970.484298:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767970.484300:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767970.484302:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767970.484303:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767970.484307:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207888700 x1862022294257792/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767970.484314:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767970.484317:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207888700 x1862022294257792/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767992 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767970.484324:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767970.484326:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767970.484328:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767970.484330:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767970.484331:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767970.484336:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767970.484339:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767970.484343:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767970.484346:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767970.484350:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207888700 x1862022294257792/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767992 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767970.484359:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207888700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294257792:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767970.484364:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767970.484367:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767967/real 1775767970] req@ffff99420788b100 x1862022294257536/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767970.484375:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767970.484376:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767970.484379:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39948. 00000400:00000200:3.0:1775767970.484382:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f990 00000400:00000010:3.0:1775767970.484384:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f990. 00000100:00000001:3.0:1775767970.484386:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767970.484390:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff99420788b100 x1862022294257536/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767970.484398:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff99420788b100 x1862022294257536/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767970.484404:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767970.484405:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767970.484408:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767970.484409:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767970.484412:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff99420788b100 x1862022294257536/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767970.484419:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767970.484423:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff99420788b100 x1862022294257536/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767992 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767970.484430:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767970.484432:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767970.484434:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767970.484435:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767970.484437:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767970.484441:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767970.484443:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767970.484448:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767970.484450:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767970.484455:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff99420788b100 x1862022294257536/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767992 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767970.484464:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff99420788b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294257536:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767970.484469:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767970.484473:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767967/real 1775767970] req@ffff994207889f80 x1862022294257664/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767970.484480:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767970.484482:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767970.484484:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39528. 00000400:00000200:3.0:1775767970.484486:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f198 00000400:00000010:3.0:1775767970.484488:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f198. 00000100:00000001:3.0:1775767970.484491:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767970.484494:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207889f80 x1862022294257664/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767970.484503:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207889f80 x1862022294257664/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767970.484511:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767970.484512:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767970.484515:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767970.484516:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767970.484520:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207889f80 x1862022294257664/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767970.484527:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767970.484531:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207889f80 x1862022294257664/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767992 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767970.484537:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767970.484540:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767970.484541:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767970.484543:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767970.484544:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767970.484549:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767970.484552:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767970.484558:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767970.484561:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767970.484566:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207889f80 x1862022294257664/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767992 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767970.484577:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207889f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294257664:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767970.484581:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767970.484583:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767970.484586:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207888700 x1862022294257792/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767992 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767970.484592:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767970.484594:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767970.484595:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1d400. 02000000:00000001:3.0:1775767970.484599:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767970.484600:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767970.484601:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:3.0:1775767970.484603:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767970.484604:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543dc00. 02000000:00000001:3.0:1775767970.484606:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767970.484607:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767970.484608:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207888700. 00000100:00000001:3.0:1775767970.484610:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767970.484611:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767970.484612:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767970.484615:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff99420788b100 x1862022294257536/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767992 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767970.484622:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767970.484623:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767970.484625:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f000. 02000000:00000001:3.0:1775767970.484627:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767970.484628:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767970.484629:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:3.0:1775767970.484631:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767970.484632:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543e400. 02000000:00000001:3.0:1775767970.484635:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767970.484636:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767970.484638:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff99420788b100. 00000100:00000001:3.0:1775767970.484640:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767970.484641:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767970.484643:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767970.484646:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207889f80 x1862022294257664/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767992 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767970.484652:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767970.484654:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767970.484655:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1e800. 02000000:00000001:3.0:1775767970.484658:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767970.484659:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767970.484661:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:3.0:1775767970.484663:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767970.484665:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543d800. 02000000:00000001:3.0:1775767970.484667:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767970.484668:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767970.484670:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207889f80. 00000100:00000001:3.0:1775767970.484672:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767970.484674:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767970.484676:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767970.484683:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767970.484686:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767970.484688:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767970.484689:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767970.545238:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767970.545244:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775767970.545246:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775767970.545249:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767970.545253:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775767970.545255:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767970.545256:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767970.545258:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767970.545261:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767970.545262:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767970.545270:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767970.545272:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767970.545274:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767970.545275:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767970.545277:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767970.545278:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767970.545294:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767970.545296:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767970.545298:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767970.609276:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767970.609282:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767970.609283:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767970.609288:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767970.609290:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767970.609294:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767970.609296:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767970.609298:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767970.609300:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767971.505278:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767971.505290:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767971.505293:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767971.505297:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:0.0:1775767971.505301:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775767971.505303:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767971.505305:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767971.505307:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767971.505309:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767971.505313:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:2.0:1775767971.505314:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775767971.505317:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767971.505318:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767971.505319:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767971.505325:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767971.505326:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767971.505334:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767971.505334:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:0.0:1775767971.505341:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767971.505343:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767971.505344:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767971.505349:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767971.505350:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767971.505363:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767971.505364:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767971.505368:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767971.505372:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767971.569278:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767971.569288:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767971.569295:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767971.569297:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767971.569301:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767971.569304:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767971.569307:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767971.569309:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767971.569314:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767971.569315:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767971.569340:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767971.569344:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767971.569348:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767971.633265:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767971.633273:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767971.633275:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767971.633281:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767971.633284:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767971.633289:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767971.633291:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767971.633294:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767971.633297:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767972.529291:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767972.529297:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767972.529299:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767972.529305:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767972.529312:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767972.529313:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767972.529316:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767972.529320:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767972.529322:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767972.529324:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767972.529333:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767972.529338:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775767972.529345:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775767972.529345:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767972.529351:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1775767972.529353:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775767972.529358:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767972.529359:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775767972.529362:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1775767972.529366:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775767972.529367:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767972.529373:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767972.529374:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775767972.529374:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767972.529379:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767972.529381:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767972.529385:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00080000:0.0:1775767972.529388:0:13471:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5224 00000100:00000001:2.0:1775767972.529393:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767972.529398:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767972.529401:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767972.529403:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767972.529406:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767972.529408:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767972.529421:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775767972.529426:0:13471:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775767972.529430:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775767972.529434:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775767972.529438:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775767972.529441:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775767972.529447:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775767972.529450:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:0.0:1775767972.529453:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775767972.529456:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:0.0:1775767972.529459:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:0.0:1775767972.529461:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:0.0:1775767972.529465:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775767972.529467:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775767972.529469:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775767972.529471:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775767972.529474:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:0.0:1775767972.529478:0:13471:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775767972.529481:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767972.529485:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775767972.529489:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775767972.529493:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767972.529496:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775767972.529499:0:13471:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775767972.529507:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fc380. 00000020:00000040:0.0:1775767972.529518:0:13471:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:0.0:1775767972.529523:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775767972.529526:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767972.529529:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775767972.529533:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775767972.529540:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775767972.529551:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220029800. 00000100:00000001:0.0:1775767972.529560:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767972.529573:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fc380 x1862022294257920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775767972.529586:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775767972.529591:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767972.529598:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002957419c] to pc [ptlrpcd_rcv+-1] req@ffff9942099fc380 x1862022294257920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767972.529631:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775767972.529635:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767972.529639:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775767972.529641:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775767972.529644:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775767972.529647:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775767972.529650:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5224 00000400:00000200:0.0:1775767972.529655:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767972.529658:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775767972.529661:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775767972.529664:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775767972.529667:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775767972.529669:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775767972.529672:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775767972.529675:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775767972.529676:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:0.0:1775767972.529678:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775767972.529680:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:0.0:1775767972.529683:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:0.0:1775767972.529685:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:0.0:1775767972.529689:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775767972.529691:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775767972.529693:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775767972.529695:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775767972.529698:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00080000:0.0:1775767972.529701:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775767972.529704:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767972.529706:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775767972.529718:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775767972.529721:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767972.529724:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775767972.529732:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775767972.529736:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767972.529739:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775767972.529746:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fed80. 00000020:00000040:0.0:1775767972.529748:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:0.0:1775767972.529752:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775767972.529753:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767972.529755:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775767972.529757:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775767972.529759:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775767972.529764:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002a800. 00000100:00000001:0.0:1775767972.529768:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767972.529774:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fed80 x1862022294258048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775767972.529782:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775767972.529792:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767972.529796:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000c09b4d8] to pc [ptlrpcd_rcv+-1] req@ffff9942099fed80 x1862022294258048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767972.529806:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775767972.529809:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767972.529813:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775767972.529815:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775767972.529817:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775767972.529821:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775767972.529824:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5224 00000400:00000200:0.0:1775767972.529828:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767972.529832:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775767972.529834:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775767972.529837:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775767972.529839:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775767972.529841:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775767972.529844:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775767972.529846:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775767972.529848:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:0.0:1775767972.529850:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775767972.529852:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:0.0:1775767972.529854:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:0.0:1775767972.529856:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:0.0:1775767972.529859:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775767972.529862:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775767972.529864:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775767972.529866:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775767972.529869:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:0.0:1775767972.529871:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775767972.529874:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767972.529875:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775767972.529878:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775767972.529881:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767972.529883:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775767972.529885:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775767972.529887:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767972.529889:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775767972.529893:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fd180. 00000020:00000040:0.0:1775767972.529895:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:0.0:1775767972.529898:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775767972.529899:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767972.529901:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775767972.529903:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775767972.529904:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775767972.529910:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002b800. 00000100:00000001:0.0:1775767972.529913:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767972.529917:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fd180 x1862022294258176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775767972.529925:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775767972.529928:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767972.529932:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004ec16d17] to pc [ptlrpcd_rcv+-1] req@ffff9942099fd180 x1862022294258176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767972.529940:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775767972.529943:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767972.529947:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775767972.529949:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775767972.529951:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775767972.529954:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775767972.529957:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5224 00000400:00000200:0.0:1775767972.529960:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767972.529963:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775767972.529966:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775767972.529969:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775767972.529972:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775767972.529974:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775767972.529976:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775767972.529978:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775767972.529979:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:0.0:1775767972.529981:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775767972.529983:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:0.0:1775767972.529986:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:0.0:1775767972.529988:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:0.0:1775767972.529991:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775767972.529993:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775767972.529995:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775767972.529997:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775767972.530000:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:0.0:1775767972.530003:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775767972.530006:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767972.530009:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775767972.530013:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775767972.530015:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767972.530017:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775767972.530020:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:0.0:1775767972.530022:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767972.530024:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775767972.530028:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ff480. 00000020:00000040:0.0:1775767972.530030:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:0.0:1775767972.530033:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775767972.530034:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767972.530036:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775767972.530038:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775767972.530039:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775767972.530043:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220029400. 00000100:00000001:0.0:1775767972.530047:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767972.530052:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ff480 x1862022294258304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775767972.530061:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775767972.530063:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767972.530067:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e73dcd12] to pc [ptlrpcd_rcv+-1] req@ffff9942099ff480 x1862022294258304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767972.530075:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775767972.530078:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767972.530082:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775767972.530084:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775767972.530086:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775767972.530088:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775767972.530091:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5224 00000400:00000200:0.0:1775767972.530095:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767972.530098:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775767972.530100:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775767972.530104:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775767972.530106:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775767972.530109:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775767972.530112:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775767972.530114:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775767972.530115:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:0.0:1775767972.530117:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775767972.530119:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:0.0:1775767972.530121:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:0.0:1775767972.530123:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:0.0:1775767972.530126:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775767972.530128:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775767972.530130:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775767972.530133:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775767972.530135:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:0.0:1775767972.530138:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775767972.530140:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767972.530142:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775767972.530146:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775767972.530148:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767972.530150:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775767972.530198:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:0.0:1775767972.530202:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767972.530204:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775767972.530208:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fd880. 00000020:00000040:0.0:1775767972.530211:0:13471:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:0.0:1775767972.530215:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775767972.530216:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767972.530218:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775767972.530220:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775767972.530222:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775767972.530226:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220028400. 00000100:00000001:0.0:1775767972.530231:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767972.530236:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fd880 x1862022294258432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775767972.530244:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775767972.530246:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767972.530256:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007f8a5f3b] to pc [ptlrpcd_rcv+-1] req@ffff9942099fd880 x1862022294258432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767972.530266:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767972.530272:0:13471:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5235) 00000100:00000001:3.0:1775767972.530492:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767972.530497:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767972.530500:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767972.530508:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fc380 x1862022294257920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767972.530517:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767972.530519:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767972.530523:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767972.530526:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767972.530530:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767972.530533:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294257920:192.168.202.156@tcp:250:kworker.0 00000100:00000001:3.0:1775767972.530538:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767972.530544:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767972.530547:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767972.530551:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767972.530558:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b1000. 02000000:00000001:3.0:1775767972.530560:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767972.530568:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767972.530573:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767972.530582:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767972.530593:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294257920, portal 25 00000100:00000001:3.0:1775767972.530596:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767972.530598:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641197440 : -112966068354176 : ffff9942099fc380) 00000100:00000040:3.0:1775767972.530605:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fc380 x1862022294257920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767997 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767972.530613:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767972.530616:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767972.530620:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767972.530626:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294257920, offset 0 00000400:00000200:3.0:1775767972.530633:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767972.530643:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767972.530650:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767972.530657:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767972.530663:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767972.530668:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767972.530672:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767972.530677:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225188:225188:256:4294967295] 192.168.202.156@tcp LPNI seq info [225184:225184:8:4294967295] 00000400:00000200:3.0:1775767972.530681:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767972.530696:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767972.530703:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767972.530709:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767972.530719:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767972.530724:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775767972.530733:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767972.530739:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000100:3.0:1775767972.530745:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767972.530748:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767972.530755:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767972.530761:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767972.530769:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767972.530773:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767972.530779:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767972.530783:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fc380 x1862022294257920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767997 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767972.530794:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767972.530798:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fc380 x1862022294257920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775767997 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767972.530806:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767972.530808:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767972.530817:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767972.530823:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767972.530826:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299898686 00000100:00000010:3.0:1775767972.530833:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207981d80. 00000100:00000001:3.0:1775767972.530843:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1775767972.530853:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299898686 00000100:00000010:3.0:1775767972.530857:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207981240. 00000100:00000001:3.0:1775767972.530861:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1775767972.530865:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767972.530869:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767972.530872:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767972.530877:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767972/real 1775767972] req@ffff9942099fc380 x1862022294257920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767972.530889:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767972.530892:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767972.530895:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767972.530898:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767972.530899:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767972.530902:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767972.530906:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fc380 x1862022294257920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767972.530916:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fc380 x1862022294257920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767972.530924:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767972.530926:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767972.530929:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767972.530931:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767972.530935:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fc380 x1862022294257920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767972.530942:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767972.530950:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fc380 x1862022294257920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767997 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767972.530959:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767972.530962:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767972.530964:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767972.530965:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767972.530968:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767972.530971:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767972.530976:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767972.530982:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767972.530985:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767972.530994:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fc380 x1862022294257920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767997 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767972.531003:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294257920:192.168.202.156@tcp:250:kworker.0 00000100:00000001:3.0:1775767972.531013:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767972.531016:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fed80 x1862022294258048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767972.531023:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767972.531025:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767972.531027:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767972.531030:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767972.531032:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767972.531035:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294258048:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767972.531038:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767972.531040:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767972.531042:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767972.531045:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767972.531049:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b3400. 02000000:00000001:3.0:1775767972.531051:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767972.531055:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767972.531057:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767972.531061:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767972.531064:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294258048, portal 10 00000100:00000001:3.0:1775767972.531069:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767972.531070:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641208192 : -112966068343424 : ffff9942099fed80) 00000100:00000040:3.0:1775767972.531078:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fed80 x1862022294258048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767997 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767972.531084:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767972.531087:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767972.531090:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767972.531093:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294258048, offset 0 00000400:00000200:3.0:1775767972.531097:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767972.531105:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767972.531111:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767972.531114:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767972.531117:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767972.531120:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767972.531123:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767972.531127:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225189:225189:256:4294967295] 192.168.202.156@tcp LPNI seq info [225185:225185:8:4294967295] 00000400:00000200:3.0:1775767972.531132:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767972.531139:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767972.531144:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767972.531148:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767972.531152:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767972.531199:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775767972.531204:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767972.531207:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000100:3.0:1775767972.531212:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767972.531214:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767972.531217:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767972.531222:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767972.531226:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767972.531228:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767972.531230:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767972.531234:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fed80 x1862022294258048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767997 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767972.531242:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767972.531245:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fed80 x1862022294258048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767997 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767972.531252:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767972.531254:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767972.531258:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767972.531262:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:3.0:1775767972.531266:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767972.531268:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299897364 00000100:00000001:3.0:1775767972.531271:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299898686 00000100:00000001:3.0:1775767972.531273:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299897364 00000100:00000001:3.0:1775767972.531276:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299898686 00000100:00000001:3.0:1775767972.531278:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767972.531281:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767972.531283:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767972.531286:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767972/real 1775767972] req@ffff9942099fed80 x1862022294258048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767972.531294:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767972.531296:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767972.531298:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767972.531300:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767972.531302:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767972.531304:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767972.531307:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fed80 x1862022294258048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767972.531316:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fed80 x1862022294258048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767972.531322:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767972.531323:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767972.531325:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767972.531326:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767972.531330:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fed80 x1862022294258048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767972.531337:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767972.531340:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fed80 x1862022294258048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767997 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767972.531346:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767972.531349:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767972.531350:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767972.531352:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767972.531354:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767972.531357:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767972.531360:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767972.531364:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767972.531366:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767972.531370:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fed80 x1862022294258048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767997 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767972.531377:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294258048:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767972.531382:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767972.531386:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fd180 x1862022294258176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767972.531392:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767972.531394:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767972.531396:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767972.531398:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767972.531400:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767972.531403:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294258176:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767972.531407:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767972.531409:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767972.531411:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767972.531413:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767972.531417:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b0400. 02000000:00000001:3.0:1775767972.531419:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767972.531423:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767972.531426:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767972.531430:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767972.531433:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294258176, portal 10 00000100:00000001:3.0:1775767972.531434:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767972.531436:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641201024 : -112966068350592 : ffff9942099fd180) 00000100:00000040:3.0:1775767972.531441:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fd180 x1862022294258176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767997 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767972.531448:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767972.531450:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767972.531452:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767972.531455:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294258176, offset 0 00000400:00000200:3.0:1775767972.531459:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767972.531463:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767972.531467:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767972.531470:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767972.531473:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767972.531476:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767972.531478:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767972.531481:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225190:225190:256:4294967295] 192.168.202.156@tcp LPNI seq info [225186:225186:8:4294967295] 00000400:00000200:3.0:1775767972.531485:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767972.531492:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767972.531497:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767972.531501:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767972.531505:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767972.531507:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775767972.531511:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767972.531514:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000100:3.0:1775767972.531518:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767972.531520:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767972.531523:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767972.531527:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767972.531530:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767972.531531:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767972.531533:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767972.531537:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fd180 x1862022294258176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767997 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767972.531545:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767972.531548:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fd180 x1862022294258176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775767997 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767972.531559:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767972.531560:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767972.531563:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767972.531567:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:3.0:1775767972.531570:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767972.531572:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299897356 00000100:00000001:3.0:1775767972.531574:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299898686 00000100:00000001:3.0:1775767972.531576:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299897356 00000100:00000001:3.0:1775767972.531578:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299898686 00000100:00000001:3.0:1775767972.531580:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767972.531582:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767972.531584:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767972.531587:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767972/real 1775767972] req@ffff9942099fd180 x1862022294258176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767972.531594:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767972.531596:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767972.531597:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767972.531599:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767972.531601:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767972.531603:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767972.531606:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fd180 x1862022294258176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767972.531768:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fd180 x1862022294258176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767972.531776:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767972.531777:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767972.531779:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767972.531781:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767972.531785:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fd180 x1862022294258176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767972.531792:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767972.531795:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fd180 x1862022294258176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767997 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767972.531802:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767972.531804:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767972.531806:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767972.531807:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767972.531809:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767972.531814:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767972.531816:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767972.531820:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767972.531822:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767972.531826:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fd180 x1862022294258176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767997 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767972.531834:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294258176:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767972.531839:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767972.531843:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ff480 x1862022294258304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767972.531850:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767972.531852:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767972.531854:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767972.531857:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767972.531859:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767972.531862:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294258304:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767972.531865:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767972.531868:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767972.531871:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767972.531872:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767972.531877:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b0000. 02000000:00000001:3.0:1775767972.531879:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767972.531884:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767972.531887:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767972.531891:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767972.531895:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294258304, portal 4 00000100:00000001:3.0:1775767972.531897:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767972.531899:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641209984 : -112966068341632 : ffff9942099ff480) 00000100:00000040:3.0:1775767972.531905:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ff480 x1862022294258304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767997 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767972.531913:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767972.531915:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767972.531918:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767972.531921:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294258304, offset 0 00000400:00000200:3.0:1775767972.531926:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767972.531931:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767972.531936:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767972.531939:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767972.531942:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767972.531944:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767972.531946:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767972.531949:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225191:225191:256:4294967295] 192.168.202.156@tcp LPNI seq info [225187:225187:8:4294967295] 00000400:00000200:3.0:1775767972.531954:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767972.531960:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767972.531965:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767972.531969:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767972.531973:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767972.531976:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775767972.531981:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767972.531984:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000100:3.0:1775767972.531989:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767972.531991:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767972.531994:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767972.531997:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767972.532000:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767972.532001:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767972.532003:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767972.532006:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ff480 x1862022294258304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767997 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767972.532011:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767972.532013:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ff480 x1862022294258304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767997 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767972.532018:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767972.532020:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767972.532023:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767972.532026:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:3.0:1775767972.532030:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767972.532032:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299897379 00000100:00000001:3.0:1775767972.532034:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299898686 00000100:00000001:3.0:1775767972.532036:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299897379 00000100:00000001:3.0:1775767972.532038:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299898686 00000100:00000001:3.0:1775767972.532040:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767972.532042:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767972.532044:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767972.532048:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767972/real 1775767972] req@ffff9942099ff480 x1862022294258304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767972.532056:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767972.532057:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767972.532059:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767972.532061:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767972.532063:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767972.532065:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767972.532069:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ff480 x1862022294258304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767972.532078:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ff480 x1862022294258304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767972.532085:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767972.532087:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767972.532089:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767972.532090:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767972.532094:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ff480 x1862022294258304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767972.532102:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767972.532105:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ff480 x1862022294258304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767997 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767972.532109:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767972.532111:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767972.532113:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767972.532113:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767972.532115:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767972.532118:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767972.532121:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767972.532123:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767972.532125:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767972.532127:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ff480 x1862022294258304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767997 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767972.532133:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294258304:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767972.532137:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767972.532140:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fd880 x1862022294258432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767972.532148:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767972.532150:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767972.532153:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767972.532230:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767972.532233:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767972.532236:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294258432:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767972.532238:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767972.532240:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767972.532241:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767972.532243:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767972.532246:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b2800. 02000000:00000001:3.0:1775767972.532247:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767972.532250:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767972.532251:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767972.532254:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767972.532256:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294258432, portal 4 00000100:00000001:3.0:1775767972.532257:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767972.532258:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641202816 : -112966068348800 : ffff9942099fd880) 00000100:00000040:3.0:1775767972.532263:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fd880 x1862022294258432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767997 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767972.532267:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767972.532269:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767972.532270:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767972.532272:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294258432, offset 0 00000400:00000200:3.0:1775767972.532275:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767972.532278:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767972.532280:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767972.532282:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767972.532284:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767972.532286:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767972.532287:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767972.532290:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225192:225192:256:4294967295] 192.168.202.156@tcp LPNI seq info [225188:225188:8:4294967295] 00000400:00000200:3.0:1775767972.532293:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767972.532298:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767972.532301:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767972.532304:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767972.532307:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767972.532309:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775767972.532312:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767972.532316:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000100:3.0:1775767972.532319:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767972.532321:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767972.532323:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767972.532326:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767972.532329:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767972.532330:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767972.532334:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767972.532340:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fd880 x1862022294258432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767997 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767972.532347:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767972.532350:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fd880 x1862022294258432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775767997 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767972.532355:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767972.532356:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767972.532358:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767972.532362:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:3.0:1775767972.532367:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767972.532378:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299897383 00000100:00000001:3.0:1775767972.532381:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299898686 00000100:00000001:3.0:1775767972.532383:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299897383 00000100:00000001:3.0:1775767972.532385:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299898686 00000100:00000001:3.0:1775767972.532387:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767972.532390:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767972.532392:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767972.532397:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767972/real 1775767972] req@ffff9942099fd880 x1862022294258432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767972.532407:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767972.532409:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767972.532411:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767972.532414:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767972.532416:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767972.532419:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767972.532422:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fd880 x1862022294258432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767972.532430:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fd880 x1862022294258432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767972.532435:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767972.532436:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767972.532438:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767972.532438:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767972.532441:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fd880 x1862022294258432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767972.532447:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767972.532452:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fd880 x1862022294258432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767997 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767972.532457:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767972.532459:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767972.532461:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767972.532463:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767972.532464:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767972.532468:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767972.532470:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767972.532473:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767972.532474:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767972.532476:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fd880 x1862022294258432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767997 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767972.532481:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294258432:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767972.532484:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767972.532486:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767972.532488:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fc380 x1862022294257920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767997 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767972.532493:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767972.532496:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767972.532498:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b1000. 02000000:00000001:3.0:1775767972.532500:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767972.532501:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767972.532503:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:3.0:1775767972.532504:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767972.532506:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029800. 02000000:00000001:3.0:1775767972.532511:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767972.532512:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767972.532514:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fc380. 00000100:00000001:3.0:1775767972.532518:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767972.532519:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767972.532521:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767972.532523:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fed80 x1862022294258048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767997 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767972.532528:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767972.532529:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767972.532530:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b3400. 02000000:00000001:3.0:1775767972.532531:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767972.532532:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767972.532533:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:3.0:1775767972.532535:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767972.532537:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002a800. 02000000:00000001:3.0:1775767972.532541:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767972.532542:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767972.532543:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fed80. 00000100:00000001:3.0:1775767972.532545:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767972.532546:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767972.532547:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767972.532549:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fd180 x1862022294258176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767997 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767972.532552:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767972.532553:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767972.532554:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b0400. 02000000:00000001:3.0:1775767972.532556:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767972.532557:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767972.532558:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:3.0:1775767972.532559:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767972.532560:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002b800. 02000000:00000001:3.0:1775767972.532561:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767972.532562:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767972.532563:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fd180. 00000100:00000001:3.0:1775767972.532564:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767972.532565:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767972.532568:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767972.532570:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ff480 x1862022294258304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767997 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767972.532575:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767972.532576:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767972.532577:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b0000. 02000000:00000001:3.0:1775767972.532578:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767972.532579:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767972.532580:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:3.0:1775767972.532581:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767972.532582:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029400. 02000000:00000001:3.0:1775767972.532584:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767972.532586:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767972.532588:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ff480. 00000100:00000001:3.0:1775767972.532592:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767972.532594:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767972.532596:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767972.532598:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fd880 x1862022294258432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767997 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767972.532602:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767972.532604:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767972.532607:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b2800. 02000000:00000001:3.0:1775767972.532609:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767972.532610:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767972.532611:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:3.0:1775767972.532612:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767972.532613:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028400. 02000000:00000001:3.0:1775767972.532614:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767972.532615:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767972.532616:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fd880. 00000100:00000001:3.0:1775767972.532617:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767972.532618:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767972.532619:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767972.532628:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767972.532630:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767972.532632:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767972.532633:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767972.593301:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767972.593306:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767972.593310:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767972.593311:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767972.593313:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767972.593314:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767972.593316:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767972.593317:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767972.593319:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767972.593320:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767972.593340:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767972.593342:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767972.593344:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767972.657292:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767972.657297:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767972.657299:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767972.657306:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767972.657310:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767972.657314:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767972.657317:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767972.657320:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767972.657323:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767973.553266:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767973.553274:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767973.553274:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767973.553277:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767973.553277:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767973.553279:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767973.553284:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775767973.553284:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767973.553288:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767973.553288:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767973.553292:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767973.553292:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767973.553296:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767973.553298:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767973.553301:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767973.553302:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767973.553303:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767973.553305:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767973.553382:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767973.553386:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767973.553387:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767973.553391:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767973.553394:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767973.553396:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767973.553398:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767973.553400:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767973.553402:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767973.617254:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767973.617260:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767973.617266:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767973.617267:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767973.617270:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767973.617271:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767973.617274:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767973.617274:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767973.617277:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767973.617278:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767973.617308:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767973.617310:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767973.617311:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767973.681223:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767973.681229:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767973.681230:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767973.681235:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767973.681238:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767973.681241:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767973.681243:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767973.681245:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767973.681247:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767973.873242:0:14953:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4299898686, at 4299898688 00000100:00000040:1.0:1775767973.873242:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299898686, at 4299898688 00000100:00000010:1.0:1775767973.873255:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207981240. 00000100:00000010:0.0:1775767973.873257:0:14953:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207981d80. 00000100:00000001:2.0:1775767974.577240:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767974.577240:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767974.577242:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767974.577247:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767974.577248:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767974.577248:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767974.577251:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767974.577251:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767974.577253:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767974.577258:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775767974.577258:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:2.0:1775767974.577259:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767974.577263:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767974.577265:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767974.577266:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767974.577268:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767974.577271:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767974.577271:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767974.577271:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767974.577275:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767974.577279:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767974.577280:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767974.577283:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767974.577286:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767974.577288:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767974.577292:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767974.577295:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767974.641242:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767974.641247:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767974.641251:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767974.641252:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767974.641254:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767974.641255:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767974.641257:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767974.641259:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767974.641261:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767974.641262:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767974.641277:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767974.641279:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767974.641281:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767974.705271:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767974.705276:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767974.705278:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767974.705282:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767974.705285:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767974.705289:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767974.705291:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767974.705293:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767974.705295:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767975.601275:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767975.601275:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767975.601275:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767975.601282:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767975.601283:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767975.601283:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767975.601284:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767975.601287:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767975.601287:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767975.601289:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:2.0:1775767975.601294:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:1.0:1775767975.601295:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767975.601295:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767975.601300:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767975.601301:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767975.601301:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767975.601305:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767975.601307:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767975.601308:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767975.601308:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767975.601315:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767975.601315:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767975.601318:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767975.601322:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767975.601323:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767975.601326:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767975.601329:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767975.665248:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767975.665252:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767975.665257:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767975.665259:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775767975.665261:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775767975.665264:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767975.665266:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775767975.665267:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767975.665269:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767975.665271:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767975.665273:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767975.665274:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775767975.665275:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775767975.665276:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767975.665277:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775767975.665278:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767975.665279:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767975.665280:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767975.665282:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767975.665284:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775767975.665284:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775767975.665285:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767975.665286:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775767975.665287:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767975.665288:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767975.665289:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767975.665291:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767975.665292:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775767975.665293:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775767975.665294:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767975.665295:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775767975.665296:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767975.665297:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767975.665299:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767975.665314:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767975.665315:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767975.665317:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767975.729255:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767975.729261:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767975.729263:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767975.729267:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767975.729270:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767975.729273:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767975.729275:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767975.729277:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767975.729279:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767976.625226:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767976.625226:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767976.625226:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767976.625232:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767976.625233:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767976.625233:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767976.625234:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767976.625236:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767976.625236:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767976.625240:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:1.0:1775767976.625241:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775767976.625241:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767976.625246:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767976.625247:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767976.625247:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767976.625251:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767976.625252:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767976.625252:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767976.625256:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767976.625258:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767976.625260:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767976.625263:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767976.625266:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767976.625269:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767976.625269:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767976.625271:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767976.625273:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767976.689242:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767976.689247:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767976.689252:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767976.689254:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767976.689257:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767976.689258:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767976.689260:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767976.689261:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767976.689264:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767976.689265:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767976.689285:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767976.689287:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767976.689289:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767976.753250:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767976.753255:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767976.753257:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767976.753261:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767976.753263:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767976.753266:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767976.753268:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767976.753270:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767976.753272:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767977.649251:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767977.649251:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767977.649252:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767977.649259:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767977.649259:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767977.649260:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767977.649262:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767977.649263:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767977.649264:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767977.649270:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775767977.649270:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:1.0:1775767977.649271:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775767977.649277:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767977.649277:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767977.649281:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767977.649281:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767977.649282:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767977.649286:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767977.649287:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767977.649290:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767977.649293:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767977.649293:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767977.649296:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767977.649298:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767977.649298:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767977.649301:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767977.649304:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775767977.649313:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767977.649320:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775767977.649324:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775767977.649328:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775767977.649333:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775767977.649340:0:13471:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5230 00000400:00000200:0.0:1775767977.649350:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767977.649358:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775767977.649363:0:13471:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775767977.649367:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775767977.649371:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775767977.649373:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775767977.649376:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775767977.649381:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775767977.649383:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:0.0:1775767977.649387:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775767977.649390:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:0.0:1775767977.649393:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:0.0:1775767977.649395:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:0.0:1775767977.649399:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775767977.649402:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775767977.649405:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775767977.649408:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775767977.649411:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:0.0:1775767977.649414:0:13471:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775767977.649418:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767977.649421:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775767977.649426:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775767977.649429:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767977.649432:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775767977.649435:0:13471:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775767977.649441:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fe300. 00000020:00000040:0.0:1775767977.649445:0:13471:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:0.0:1775767977.649451:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775767977.649453:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767977.649456:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775767977.649458:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775767977.649463:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775767977.649471:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002bc00. 00000100:00000001:0.0:1775767977.649478:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767977.649488:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fe300 x1862022294258560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775767977.649500:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775767977.649504:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767977.649510:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008a1e3639] to pc [ptlrpcd_rcv+-1] req@ffff9942099fe300 x1862022294258560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767977.649530:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775767977.649533:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767977.649537:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775767977.649539:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775767977.649540:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775767977.649543:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775767977.649546:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5230 00000400:00000200:0.0:1775767977.649551:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767977.649554:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775767977.649556:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775767977.649559:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775767977.649561:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775767977.649563:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775767977.649565:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775767977.649567:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775767977.649568:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:0.0:1775767977.649570:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775767977.649572:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:0.0:1775767977.649574:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:0.0:1775767977.649576:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:0.0:1775767977.649579:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775767977.649582:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775767977.649583:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775767977.649585:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775767977.649588:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00080000:0.0:1775767977.649591:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775767977.649593:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767977.649595:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775767977.649600:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775767977.649602:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767977.649604:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775767977.649608:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775767977.649611:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767977.649613:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775767977.649617:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ff100. 00000020:00000040:0.0:1775767977.649619:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:0.0:1775767977.649622:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775767977.649623:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767977.649625:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775767977.649626:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775767977.649628:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775767977.649631:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220028000. 00000100:00000001:0.0:1775767977.649634:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767977.649639:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ff100 x1862022294258688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775767977.649657:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775767977.649663:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767977.649668:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5e24158] to pc [ptlrpcd_rcv+-1] req@ffff9942099ff100 x1862022294258688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767977.649683:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775767977.649686:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767977.649689:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775767977.649691:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775767977.649692:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775767977.649695:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775767977.649698:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5230 00000400:00000200:0.0:1775767977.649703:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767977.649706:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775767977.649709:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775767977.649711:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775767977.649714:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775767977.649717:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775767977.649719:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775767977.649721:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775767977.649723:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:0.0:1775767977.649725:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775767977.649727:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:0.0:1775767977.649729:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:0.0:1775767977.649732:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000001:3.0:1775767977.649734:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1775767977.649735:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767977.649738:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767977.649739:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1775767977.649740:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775767977.649743:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775767977.649748:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000040:3.0:1775767977.649748:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fe300 x1862022294258560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1775767977.649753:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:0.0:1775767977.649758:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775767977.649760:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767977.649763:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775767977.649766:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775767977.649769:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767977.649771:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775767977.649773:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775767977.649776:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767977.649778:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775767977.649782:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fc700. 00000020:00000040:0.0:1775767977.649785:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:0.0:1775767977.649787:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775767977.649789:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767977.649791:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775767977.649793:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775767977.649796:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775767977.649802:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220028400. 00000100:00000001:0.0:1775767977.649807:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767977.649813:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00100000:0.0:1775767977.649816:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fc700 x1862022294258816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767977.649817:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767977.649824:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767977.649827:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775767977.649829:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1775767977.649830:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767977.649833:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767977.649837:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294258560:192.168.202.156@tcp:250:kworker.0 00000100:00000040:0.0:1775767977.649842:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003283b20a] to pc [ptlrpcd_rcv+-1] req@ffff9942099fc700 x1862022294258816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767977.649843:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767977.649847:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767977.649851:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767977.649856:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1775767977.649858:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767977.649861:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b2c00. 00000100:00080000:0.0:1775767977.649862:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:3.0:1775767977.649864:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775767977.649869:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775767977.649872:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:3.0:1775767977.649873:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00080000:0.0:1775767977.649876:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000010:3.0:1775767977.649878:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000100:00000001:0.0:1775767977.649882:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000400:00000010:3.0:1775767977.649884:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00080000:0.0:1775767977.649886:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5230 00000100:00000200:3.0:1775767977.649891:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294258560, portal 25 00000400:00000200:0.0:1775767977.649892:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767977.649895:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000400:00000200:0.0:1775767977.649899:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:3.0:1775767977.649899:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641205504 : -112966068346112 : ffff9942099fe300) 00000100:00080000:0.0:1775767977.649904:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767977.649908:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fe300 x1862022294258560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768002 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775767977.649911:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775767977.649917:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1775767977.649920:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775767977.649922:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775767977.649927:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000040:3.0:1775767977.649927:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767977.649930:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000020:00000001:0.0:1775767977.649931:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775767977.649934:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000100:00000200:3.0:1775767977.649935:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294258560, offset 0 00000020:00000001:0.0:1775767977.649938:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775767977.649941:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000400:00000200:3.0:1775767977.649941:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000020:00000001:0.0:1775767977.649946:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000400:00000200:3.0:1775767977.649950:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:0.0:1775767977.649951:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:0.0:1775767977.649958:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000200:3.0:1775767977.649958:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00000001:0.0:1775767977.649963:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1775767977.649965:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:0.0:1775767977.649969:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000200:3.0:1775767977.649971:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000100:00000001:0.0:1775767977.649975:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000200:3.0:1775767977.649977:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000020:00000040:0.0:1775767977.649980:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000400:00000200:3.0:1775767977.649981:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00080000:0.0:1775767977.649988:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000400:00000200:3.0:1775767977.649989:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225193:225193:256:4294967295] 192.168.202.156@tcp LPNI seq info [225189:225189:8:4294967295] 00000100:00000001:0.0:1775767977.649993:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767977.649998:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:0.0:1775767977.649999:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775767977.650003:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775767977.650008:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767977.650010:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000100:00000001:0.0:1775767977.650012:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775767977.650017:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000800:00000200:3.0:1775767977.650019:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000008:00000001:0.0:1775767977.650022:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1775767977.650024:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000100:00000001:0.0:1775767977.650025:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775767977.650031:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fd500. 00000800:00000200:3.0:1775767977.650032:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000040:0.0:1775767977.650036:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00000800:00000200:3.0:1775767977.650037:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 02000000:00000001:0.0:1775767977.650042:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775767977.650046:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1775767977.650046:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:0.0:1775767977.650050:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000800:00000200:3.1:1775767977.650051:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 02000000:00000001:0.0:1775767977.650053:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000800:00000100:3.0:1775767977.650057:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 02000000:00000001:0.0:1775767977.650058:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1775767977.650060:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 02000000:00000010:0.0:1775767977.650063:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220029400. 00000400:00000200:3.0:1775767977.650064:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:0.0:1775767977.650070:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767977.650070:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767977.650076:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767977.650079:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00100000:0.0:1775767977.650080:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fd500 x1862022294258944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767977.650083:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767977.650089:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fe300 x1862022294258560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768002 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767977.650098:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775767977.650103:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767977.650103:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767977.650108:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fe300 x1862022294258560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768002 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775767977.650125:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f246bb2b] to pc [ptlrpcd_rcv+-1] req@ffff9942099fd500 x1862022294258944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767977.650139:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767977.650143:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775767977.650148:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767977.650149:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775767977.650151:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:3.0:1775767977.650175:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767977.650178:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299903806 00000100:00000010:3.0:1775767977.650183:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207981480. 00000100:00080000:0.0:1775767977.650184:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775767977.650187:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775767977.650189:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767977.650190:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775767977.650193:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:3.0:1775767977.650197:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299903806 00000100:00080000:0.0:1775767977.650198:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5230 00000100:00000010:3.0:1775767977.650202:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942079812a0. 00000400:00000200:0.0:1775767977.650206:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767977.650206:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1775767977.650211:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1775767977.650214:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:3.0:1775767977.650216:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775767977.650219:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1775767977.650220:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000040:0.0:1775767977.650226:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00100000:3.0:1775767977.650227:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767977/real 1775767977] req@ffff9942099fe300 x1862022294258560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767977.650231:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775767977.650237:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775767977.650242:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775767977.650247:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775767977.650250:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000400:00000001:3.0:1775767977.650251:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:0.0:1775767977.650252:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000001:3.0:1775767977.650253:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000040:0.0:1775767977.650256:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000400:00000010:3.0:1775767977.650256:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000020:00000001:0.0:1775767977.650261:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000400:00000200:3.0:1775767977.650262:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767977.650265:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000020:00000001:0.0:1775767977.650267:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000001:3.0:1775767977.650268:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:0.0:1775767977.650272:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000200:3.0:1775767977.650274:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fe300 x1862022294258560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767977.650277:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775767977.650283:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000200:3.0:1775767977.650287:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fe300 x1862022294258560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767977.650288:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775767977.650294:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00000001:3.0:1775767977.650300:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00080000:0.0:1775767977.650302:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767977.650302:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767977.650306:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767977.650307:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767977.650308:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1775767977.650312:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:3.0:1775767977.650315:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fe300 x1862022294258560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1775767977.650318:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775767977.650323:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767977.650328:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775767977.650328:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:0.0:1775767977.650331:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000100:00000040:3.0:1775767977.650333:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fe300 x1862022294258560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768002 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:0.0:1775767977.650335:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767977.650340:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767977.650346:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000010:0.0:1775767977.650347:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ff800. 00000100:00000001:3.0:1775767977.650348:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767977.650352:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000040:0.0:1775767977.650353:0:13471:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00000100:00000001:3.0:1775767977.650353:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767977.650357:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 02000000:00000001:0.0:1775767977.650359:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1775767977.650359:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:0.0:1775767977.650362:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767977.650363:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:0.0:1775767977.650367:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00080000:3.0:1775767977.650369:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 02000000:00000001:0.0:1775767977.650371:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1775767977.650372:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:0.0:1775767977.650374:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767977.650376:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fe300 x1862022294258560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768002 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1775767977.650379:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002b800. 00000100:00000001:0.0:1775767977.650386:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767977.650387:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294258560:192.168.202.156@tcp:250:kworker.0 00000100:00000001:3.0:1775767977.650392:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00100000:0.0:1775767977.650396:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ff800 x1862022294259072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:3.0:1775767977.650397:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ff100 x1862022294258688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767977.650423:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767977.650426:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767977.650427:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1775767977.650428:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767977.650431:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775767977.650433:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767977.650433:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767977.650440:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294258688:192.168.202.156@tcp:38:kworker.0 00000100:00000040:0.0:1775767977.650444:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000aa4e8b71] to pc [ptlrpcd_rcv+-1] req@ffff9942099ff800 x1862022294259072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767977.650444:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767977.650446:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767977.650448:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767977.650452:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767977.650455:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b2800. 02000000:00000001:3.0:1775767977.650458:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767977.650465:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767977.650465:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767977.650470:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000100:00000040:0.0:1775767977.650473:0:13471:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5240) 00000400:00000010:3.0:1775767977.650473:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767977.650479:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294258688, portal 10 00000100:00000001:3.0:1775767977.650481:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767977.650482:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641209088 : -112966068342528 : ffff9942099ff100) 00000100:00000040:3.0:1775767977.650489:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ff100 x1862022294258688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768002 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767977.650494:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767977.650496:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767977.650498:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767977.650500:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294258688, offset 0 00000400:00000200:3.0:1775767977.650503:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767977.650507:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767977.650511:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767977.650514:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767977.650516:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767977.650519:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767977.650521:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767977.650524:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225194:225194:256:4294967295] 192.168.202.156@tcp LPNI seq info [225190:225190:8:4294967295] 00000400:00000200:3.0:1775767977.650527:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767977.650533:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767977.650538:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767977.650542:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767977.650547:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767977.650550:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775767977.650555:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767977.650558:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000100:3.0:1775767977.650562:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767977.650565:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767977.650568:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767977.650572:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767977.650576:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767977.650578:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767977.650582:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767977.650587:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ff100 x1862022294258688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768002 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767977.650597:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767977.650602:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ff100 x1862022294258688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768002 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767977.650609:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767977.650610:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767977.650614:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767977.650617:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:3.0:1775767977.650619:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767977.650621:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299902483 00000100:00000001:3.0:1775767977.650623:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299903806 00000100:00000001:3.0:1775767977.650625:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299902483 00000100:00000001:3.0:1775767977.650627:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299903806 00000100:00000001:3.0:1775767977.650628:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767977.650630:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767977.650631:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767977.650634:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767977/real 1775767977] req@ffff9942099ff100 x1862022294258688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767977.650639:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767977.650639:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767977.650640:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767977.650642:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767977.650643:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767977.650644:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767977.650646:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ff100 x1862022294258688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767977.650651:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ff100 x1862022294258688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767977.650656:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767977.650657:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767977.650658:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767977.650659:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767977.650661:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ff100 x1862022294258688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767977.650665:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767977.650667:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ff100 x1862022294258688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768002 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767977.650671:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767977.650673:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767977.650674:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767977.650676:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767977.650677:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767977.650679:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767977.650681:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767977.650683:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767977.650684:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767977.650686:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ff100 x1862022294258688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768002 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767977.650691:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294258688:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767977.650694:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767977.650696:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767977.650698:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fe300 x1862022294258560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768002 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767977.650702:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767977.650704:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767977.650706:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b2c00. 02000000:00000001:3.0:1775767977.650707:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767977.650708:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767977.650709:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:3.0:1775767977.650710:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767977.650712:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002bc00. 02000000:00000001:3.0:1775767977.650714:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767977.650716:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767977.650717:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fe300. 00000100:00000001:3.0:1775767977.650719:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767977.650720:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767977.650722:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767977.650724:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ff100 x1862022294258688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768002 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767977.650732:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767977.650733:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767977.650734:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b2800. 02000000:00000001:3.0:1775767977.650736:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767977.650736:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767977.650737:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:3.0:1775767977.650738:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767977.650739:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028000. 02000000:00000001:3.0:1775767977.650740:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767977.650741:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767977.650742:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ff100. 00000100:00000001:3.0:1775767977.650743:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767977.650744:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767977.650745:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767977.650750:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767977.650751:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767977.650752:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767977.650753:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767977.650754:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767977.650758:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fc700 x1862022294258816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767977.650764:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767977.650766:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767977.650768:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767977.650769:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767977.650771:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767977.650774:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294258816:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767977.650777:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767977.650779:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767977.650781:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767977.650783:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767977.650786:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b2800. 02000000:00000001:3.0:1775767977.650788:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767977.650792:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767977.650795:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767977.650798:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767977.650801:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294258816, portal 10 00000100:00000001:3.0:1775767977.650804:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767977.650805:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641198336 : -112966068353280 : ffff9942099fc700) 00000100:00000040:3.0:1775767977.650811:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fc700 x1862022294258816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768002 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767977.650817:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767977.650819:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767977.650821:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767977.650823:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294258816, offset 0 00000400:00000200:3.0:1775767977.650825:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767977.650829:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767977.650831:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767977.650834:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767977.650836:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767977.650838:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767977.650839:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767977.650841:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225195:225195:256:4294967295] 192.168.202.156@tcp LPNI seq info [225191:225191:8:4294967295] 00000400:00000200:3.0:1775767977.650845:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767977.650849:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767977.650852:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767977.650854:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767977.650857:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767977.650859:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775767977.650862:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767977.650864:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000100:3.0:1775767977.650866:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767977.650868:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767977.650870:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767977.650872:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767977.650874:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767977.650876:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767977.650878:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767977.650880:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fc700 x1862022294258816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768002 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767977.650886:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767977.650888:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fc700 x1862022294258816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768002 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767977.650892:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767977.650893:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767977.650895:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767977.650897:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:3.0:1775767977.650898:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767977.650900:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299902475 00000100:00000001:3.0:1775767977.650901:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299903806 00000100:00000001:3.0:1775767977.650903:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299902475 00000100:00000001:3.0:1775767977.650904:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299903806 00000100:00000001:3.0:1775767977.650906:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767977.650907:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767977.650909:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767977.650911:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767977/real 1775767977] req@ffff9942099fc700 x1862022294258816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767977.650915:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767977.650916:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767977.650917:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767977.650918:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767977.650919:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767977.650920:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767977.650922:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fc700 x1862022294258816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767977.650927:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fc700 x1862022294258816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767977.650932:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767977.650933:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767977.650935:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767977.650936:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767977.650938:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fc700 x1862022294258816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767977.650942:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767977.650944:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fc700 x1862022294258816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768002 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767977.650948:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767977.650949:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767977.650950:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767977.650951:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767977.650952:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767977.650954:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767977.650955:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767977.650957:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767977.650959:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767977.650961:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fc700 x1862022294258816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768002 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767977.650966:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294258816:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767977.650968:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767977.650970:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fd500 x1862022294258944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767977.650975:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767977.650976:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767977.650978:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767977.650979:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767977.650980:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767977.650982:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294258944:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767977.650985:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767977.650986:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767977.650987:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767977.650988:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767977.650989:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b2c00. 02000000:00000001:3.0:1775767977.650991:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767977.650993:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767977.650995:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767977.650997:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767977.650999:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294258944, portal 4 00000100:00000001:3.0:1775767977.651000:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767977.651001:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641201920 : -112966068349696 : ffff9942099fd500) 00000100:00000040:3.0:1775767977.651005:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fd500 x1862022294258944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768002 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767977.651009:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767977.651010:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767977.651011:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767977.651013:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294258944, offset 0 00000400:00000200:3.0:1775767977.651015:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767977.651017:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767977.651020:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767977.651022:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767977.651023:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767977.651025:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767977.651026:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767977.651029:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225196:225196:256:4294967295] 192.168.202.156@tcp LPNI seq info [225192:225192:8:4294967295] 00000400:00000200:3.0:1775767977.651032:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767977.651036:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767977.651039:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767977.651041:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767977.651044:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767977.651045:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775767977.651048:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767977.651050:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000100:3.0:1775767977.651052:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767977.651054:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767977.651056:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767977.651058:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767977.651059:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767977.651061:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767977.651062:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767977.651064:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fd500 x1862022294258944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768002 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767977.651069:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767977.651071:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fd500 x1862022294258944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768002 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767977.651076:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767977.651077:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767977.651078:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767977.651080:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:3.0:1775767977.651082:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767977.651083:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299902498 00000100:00000001:3.0:1775767977.651084:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299903806 00000100:00000001:3.0:1775767977.651085:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299902498 00000100:00000001:3.0:1775767977.651087:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299903806 00000100:00000001:3.0:1775767977.651089:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767977.651090:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767977.651092:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767977.651094:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767977/real 1775767977] req@ffff9942099fd500 x1862022294258944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767977.651100:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767977.651101:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767977.651102:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767977.651103:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767977.651104:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767977.651106:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767977.651107:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fd500 x1862022294258944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767977.651127:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fd500 x1862022294258944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767977.651131:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767977.651132:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767977.651133:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767977.651134:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767977.651136:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fd500 x1862022294258944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767977.651141:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767977.651143:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fd500 x1862022294258944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768002 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767977.651148:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767977.651149:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767977.651150:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767977.651151:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767977.651152:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767977.651171:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767977.651174:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767977.651176:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767977.651178:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767977.651182:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fd500 x1862022294258944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768002 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767977.651188:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294258944:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767977.651192:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767977.651195:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ff800 x1862022294259072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767977.651200:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767977.651201:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767977.651202:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767977.651203:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767977.651205:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767977.651207:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294259072:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767977.651209:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767977.651210:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767977.651211:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767977.651212:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767977.651215:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b0000. 02000000:00000001:3.0:1775767977.651216:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767977.651218:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767977.651220:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767977.651222:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767977.651224:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294259072, portal 4 00000100:00000001:3.0:1775767977.651226:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767977.651227:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641210880 : -112966068340736 : ffff9942099ff800) 00000100:00000040:3.0:1775767977.651230:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ff800 x1862022294259072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768002 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767977.651235:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767977.651236:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767977.651238:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767977.651239:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294259072, offset 0 00000400:00000200:3.0:1775767977.651242:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767977.651245:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767977.651247:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767977.651249:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767977.651251:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767977.651253:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767977.651255:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767977.651257:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225197:225197:256:4294967295] 192.168.202.156@tcp LPNI seq info [225193:225193:8:4294967295] 00000400:00000200:3.0:1775767977.651260:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767977.651264:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767977.651267:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767977.651268:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767977.651271:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767977.651273:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775767977.651276:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767977.651278:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000100:3.0:1775767977.651280:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767977.651282:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767977.651284:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767977.651286:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767977.651288:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767977.651289:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767977.651291:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767977.651293:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ff800 x1862022294259072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768002 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767977.651298:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767977.651300:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ff800 x1862022294259072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768002 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767977.651304:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767977.651305:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767977.651306:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767977.651308:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:3.0:1775767977.651310:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767977.651312:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299902502 00000100:00000001:3.0:1775767977.651314:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299903806 00000100:00000001:3.0:1775767977.651316:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299902502 00000100:00000001:3.0:1775767977.651317:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299903806 00000100:00000001:3.0:1775767977.651318:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767977.651319:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767977.651320:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767977.651322:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767977/real 1775767977] req@ffff9942099ff800 x1862022294259072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767977.651327:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767977.651328:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767977.651329:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767977.651330:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767977.651332:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767977.651333:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767977.651335:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ff800 x1862022294259072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767977.651340:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ff800 x1862022294259072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767977.651344:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767977.651345:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767977.651346:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767977.651346:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767977.651348:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ff800 x1862022294259072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767977.651352:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767977.651354:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ff800 x1862022294259072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768002 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767977.651358:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767977.651360:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767977.651361:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767977.651362:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767977.651363:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767977.651366:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767977.651368:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767977.651370:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767977.651371:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767977.651373:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ff800 x1862022294259072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768002 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767977.651377:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294259072:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767977.651380:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767977.651381:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767977.651383:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fc700 x1862022294258816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768002 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767977.651387:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767977.651388:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767977.651389:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b2800. 02000000:00000001:3.0:1775767977.651391:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767977.651392:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767977.651393:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:3.0:1775767977.651394:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767977.651396:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028400. 02000000:00000001:3.0:1775767977.651397:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767977.651398:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767977.651399:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fc700. 00000100:00000001:3.0:1775767977.651401:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767977.651402:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767977.651403:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767977.651405:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fd500 x1862022294258944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768002 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767977.651414:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767977.651415:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767977.651416:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b2c00. 02000000:00000001:3.0:1775767977.651417:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767977.651418:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767977.651419:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:3.0:1775767977.651420:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767977.651420:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029400. 02000000:00000001:3.0:1775767977.651422:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767977.651423:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767977.651423:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fd500. 00000100:00000001:3.0:1775767977.651425:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767977.651426:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767977.651426:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767977.651428:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ff800 x1862022294259072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768002 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767977.651432:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767977.651432:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767977.651433:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b0000. 02000000:00000001:3.0:1775767977.651435:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767977.651435:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767977.651436:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:3.0:1775767977.651437:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767977.651438:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002b800. 02000000:00000001:3.0:1775767977.651439:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767977.651440:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767977.651441:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ff800. 00000100:00000001:3.0:1775767977.651442:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767977.651442:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767977.651443:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767977.651446:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767977.651446:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767977.651448:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767977.651449:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767977.713270:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767977.713276:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767977.713282:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767977.713285:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767977.713289:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767977.713291:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767977.713294:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767977.713295:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767977.713299:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767977.713300:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767977.713320:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767977.713324:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767977.713327:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767977.777246:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767977.777251:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767977.777253:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767977.777257:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767977.777260:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767977.777263:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767977.777265:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767977.777267:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767977.777269:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767978.673241:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767978.673241:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767978.673241:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767978.673248:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767978.673248:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767978.673249:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767978.673250:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767978.673253:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767978.673253:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767978.673256:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:1.0:1775767978.673261:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775767978.673261:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1775767978.673261:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767978.673265:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767978.673266:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767978.673270:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767978.673270:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767978.673272:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767978.673277:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767978.673277:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767978.673277:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767978.673283:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767978.673283:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767978.673290:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767978.673290:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767978.673294:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767978.673299:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767978.737235:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767978.737240:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767978.737243:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767978.737244:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767978.737247:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767978.737247:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767978.737250:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767978.737251:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767978.737252:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767978.737253:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767978.737268:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767978.737270:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767978.737272:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767978.801247:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767978.801254:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767978.801256:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767978.801261:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767978.801264:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767978.801268:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767978.801272:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767978.801274:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767978.801277:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767978.993249:0:14953:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4299903806, at 4299903808 00000100:00000040:1.0:1775767978.993249:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299903806, at 4299903808 00000100:00000010:0.0:1775767978.993264:0:14953:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207981480. 00000100:00000010:1.0:1775767978.993271:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942079812a0. 00000100:00000001:1.0:1775767979.697255:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767979.697256:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767979.697261:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767979.697262:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767979.697263:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767979.697267:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767979.697267:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767979.697267:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767979.697270:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767979.697274:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775767979.697274:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775767979.697275:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767979.697281:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767979.697281:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767979.697281:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767979.697287:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767979.697287:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767979.697287:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767979.697293:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767979.697296:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767979.697299:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767979.697300:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767979.697304:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767979.697308:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767979.697308:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767979.697309:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767979.697312:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767979.761254:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767979.761258:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767979.761262:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767979.761263:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767979.761266:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767979.761268:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767979.761270:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767979.761271:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767979.761274:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767979.761275:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767979.761291:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767979.761300:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767979.761302:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767979.825211:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767979.825216:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767979.825217:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767979.825222:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767979.825224:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767979.825228:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767979.825230:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767979.825232:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767979.825234:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767980.721215:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767980.721219:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767980.721221:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767980.721223:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767980.721224:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767980.721228:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767980.721229:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775767980.721233:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767980.721234:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767980.721236:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767980.721238:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767980.721239:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767980.721242:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767980.721242:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767980.721243:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767980.721248:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767980.721248:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767980.721252:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767980.721253:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00100000:2.0:1775767980.721254:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767980.721256:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767980.721260:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767980.721261:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767980.721266:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767980.721269:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767980.721271:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767980.721274:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767980.785224:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767980.785230:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775767980.785232:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775767980.785235:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767980.785239:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775767980.785241:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767980.785243:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767980.785245:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767980.785247:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767980.785248:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767980.785251:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767980.785252:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767980.785254:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767980.785255:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767980.785257:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767980.785258:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767980.785272:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767980.785274:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767980.785275:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767980.849260:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767980.849266:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767980.849267:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767980.849271:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767980.849274:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767980.849277:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767980.849279:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767980.849281:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767980.849283:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767981.745258:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767981.745263:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767981.745264:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767981.745264:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767981.745267:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767981.745270:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767981.745273:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767981.745274:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767981.745276:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767981.745280:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775767981.745281:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:2.0:1775767981.745282:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767981.745292:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767981.745292:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767981.745293:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767981.745303:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767981.745304:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767981.745305:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767981.745317:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767981.745321:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767981.745323:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767981.745326:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767981.745328:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767981.745331:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767981.745332:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767981.745336:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767981.745339:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767981.809336:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767981.809343:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767981.809351:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767981.809353:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767981.809356:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767981.809358:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767981.809361:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767981.809363:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767981.809367:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767981.809368:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767981.809418:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767981.809422:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767981.809428:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767981.873268:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767981.873275:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767981.873277:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767981.873283:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767981.873287:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767981.873292:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767981.873294:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767981.873297:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767981.873301:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767982.769224:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767982.769229:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767982.769229:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767982.769233:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767982.769234:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767982.769236:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767982.769241:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775767982.769241:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767982.769247:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767982.769247:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767982.769252:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767982.769252:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767982.769258:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767982.769260:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767982.769263:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767982.769264:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767982.769267:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767982.769271:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767982.769283:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767982.769288:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:1.0:1775767982.769289:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775767982.769290:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767982.769295:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775767982.769299:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:1.0:1775767982.769299:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775767982.769303:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767982.769304:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1775767982.769306:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767982.769309:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767982.769311:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767982.769311:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775767982.769318:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775767982.769328:0:13471:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5235 00000400:00000200:1.0:1775767982.769338:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767982.769345:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775767982.769349:0:13471:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775767982.769352:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767982.769356:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767982.769358:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767982.769361:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775767982.769365:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775767982.769367:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:1.0:1775767982.769371:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775767982.769373:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:1.0:1775767982.769376:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:1.0:1775767982.769379:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:1.0:1775767982.769382:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767982.769384:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767982.769386:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767982.769388:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775767982.769392:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:1.0:1775767982.769395:0:13471:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775767982.769399:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767982.769402:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775767982.769405:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767982.769408:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767982.769411:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775767982.769414:0:13471:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775767982.769423:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207889f80. 00000020:00000040:1.0:1775767982.769427:0:13471:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:1.0:1775767982.769431:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775767982.769433:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767982.769436:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775767982.769439:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775767982.769445:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775767982.769454:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543c800. 00000100:00000001:1.0:1775767982.769460:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767982.769471:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207889f80 x1862022294259200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775767982.769482:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775767982.769487:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767982.769492:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ed2274f4] to pc [ptlrpcd_rcv+-1] req@ffff994207889f80 x1862022294259200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767982.769514:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767982.769517:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767982.769521:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775767982.769523:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775767982.769525:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775767982.769529:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775767982.769532:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5235 00000100:00000001:3.0:1775767982.769532:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767982.769536:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767982.769539:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000400:00000200:1.0:1775767982.769541:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767982.769548:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000040:3.0:1775767982.769548:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207889f80 x1862022294259200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767982.769555:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775767982.769564:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767982.769565:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767982.769569:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767982.769570:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1775767982.769572:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:1.0:1775767982.769575:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000001:3.0:1775767982.769576:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775767982.769580:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 02000000:00000001:3.0:1775767982.769581:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775767982.769587:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000100:00100000:3.0:1775767982.769589:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207889f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294259200:192.168.202.156@tcp:250:kworker.0 00000020:00000040:1.0:1775767982.769590:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:1.0:1775767982.769594:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:3.0:1775767982.769597:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000040:1.0:1775767982.769598:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:1.0:1775767982.769602:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 02000000:00000001:3.0:1775767982.769602:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767982.769605:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775767982.769607:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 02000000:00000001:3.0:1775767982.769608:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767982.769614:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b0000. 00000100:00000040:1.0:1775767982.769615:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 02000000:00000001:3.0:1775767982.769618:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767982.769620:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767982.769624:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000200:3.0:1775767982.769626:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767982.769630:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000010:3.0:1775767982.769632:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000020:00000040:1.0:1775767982.769635:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000400:00000010:3.0:1775767982.769639:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00080000:1.0:1775767982.769640:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775767982.769645:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1775767982.769646:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294259200, portal 25 02000000:00000001:1.0:1775767982.769648:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1775767982.769650:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767982.769652:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606126464 : -112966103425152 : ffff994207889f80) 02000000:00000001:1.0:1775767982.769656:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1775767982.769657:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207889f80 x1862022294259200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768007 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1775767982.769661:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767982.769668:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775767982.769672:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00100000:1.0:1775767982.769673:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00000040:3.0:1775767982.769676:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767982.769680:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000008:00000001:1.0:1775767982.769681:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1775767982.769684:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294259200, offset 0 00000100:00000001:1.0:1775767982.769685:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767982.769694:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00000010:1.0:1775767982.769695:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff99420788b100. 00000020:00000040:1.0:1775767982.769699:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:1.0:1775767982.769703:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000200:3.0:1775767982.769703:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:1.0:1775767982.769707:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767982.769711:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:3.0:1775767982.769711:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 02000000:00000001:1.0:1775767982.769715:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:3.0:1775767982.769718:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:1.0:1775767982.769720:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767982.769723:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 02000000:00000010:1.0:1775767982.769727:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543c400. 00000400:00000200:3.0:1775767982.769727:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767982.769732:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00000001:1.0:1775767982.769735:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767982.769737:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225198:225198:256:4294967295] 192.168.202.156@tcp LPNI seq info [225194:225194:8:4294967295] 00000400:00000200:3.0:1775767982.769742:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00100000:1.0:1775767982.769746:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff99420788b100 x1862022294259328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:3.0:1775767982.769756:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767982.769770:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00000001:1.0:1775767982.769771:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775767982.769775:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1775767982.769775:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000100:00000040:1.0:1775767982.769784:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e26cf977] to pc [ptlrpcd_rcv+-1] req@ffff99420788b100 x1862022294259328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1775767982.769784:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767982.769790:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775767982.769799:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767982.769805:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000100:00000001:1.0:1775767982.769807:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1775767982.769812:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000100:00080000:1.0:1775767982.769813:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000010:3.0:1775767982.769814:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000100:00080000:1.0:1775767982.769820:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000200:3.0:1775767982.769822:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:1.0:1775767982.769823:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775767982.769827:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:3.0:1775767982.769828:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000100:00000001:1.0:1775767982.769835:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:3.0:1775767982.769835:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767982.769839:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00080000:1.0:1775767982.769843:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5235 00000100:00000001:3.0:1775767982.769843:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767982.769849:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207889f80 x1862022294259200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768007 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775767982.769854:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767982.769863:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:3.0:1775767982.769864:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00080000:1.0:1775767982.769868:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767982.769870:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207889f80 x1862022294259200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768007 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775767982.769876:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767982.769884:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1775767982.769884:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767982.769887:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:1.0:1775767982.769890:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767982.769893:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767982.769894:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775767982.769899:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000400:00000200:3.0:1775767982.769901:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000040:1.0:1775767982.769902:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000100:00000001:3.0:1775767982.769905:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299908925 00000020:00000001:1.0:1775767982.769906:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775767982.769910:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000100:00000010:3.0:1775767982.769912:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207981f60. 00000020:00000001:1.0:1775767982.769913:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:1.0:1775767982.769918:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000001:3.0:1775767982.769920:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000040:1.0:1775767982.769923:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767982.769926:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299908925 00000100:00000001:1.0:1775767982.769927:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000010:3.0:1775767982.769930:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207981900. 00000100:00000001:3.0:1775767982.769933:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1775767982.769935:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767982.769937:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767982.769939:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000040:1.0:1775767982.769943:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00100000:3.0:1775767982.769945:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767982/real 1775767982] req@ffff994207889f80 x1862022294259200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767982.769948:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775767982.769954:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:1.0:1775767982.769963:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775767982.769968:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767982.769972:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000001:3.0:1775767982.769972:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:1.0:1775767982.769976:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000001:3.0:1775767982.769976:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767982.769980:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 02000000:00000001:1.0:1775767982.769981:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767982.769986:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:3.0:1775767982.769987:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000008:00100000:1.0:1775767982.769988:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000400:00000010:3.0:1775767982.769991:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000008:00000001:1.0:1775767982.769993:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767982.769996:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1775767982.769998:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775767982.770004:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207888700. 00000100:00000200:3.0:1775767982.770004:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207889f80 x1862022294259200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1775767982.770009:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:1.0:1775767982.770015:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775767982.770018:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767982.770022:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000200:3.0:1775767982.770023:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207889f80 x1862022294259200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1775767982.770025:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775767982.770028:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767982.770034:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000010:1.0:1775767982.770035:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543f800. 00000100:00000001:3.0:1775767982.770036:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767982.770041:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775767982.770042:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767982.770043:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767982.770049:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207889f80 x1862022294259200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767982.770053:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207888700 x1862022294259456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767982.770073:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767982.770082:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207889f80 x1862022294259200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768007 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767982.770086:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775767982.770091:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775767982.770101:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009f2ba301] to pc [ptlrpcd_rcv+-1] req@ffff994207888700 x1862022294259456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767982.770121:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775767982.770124:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775767982.770130:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1775767982.770130:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767982.770136:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:1.0:1775767982.770137:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775767982.770140:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767982.770141:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775767982.770145:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767982.770146:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767982.770150:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775767982.770151:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767982.770180:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767982.770185:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5235 00000100:00080000:3.0:1775767982.770189:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767982.770194:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000400:00000200:1.0:1775767982.770195:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767982.770204:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000040:3.0:1775767982.770206:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207889f80 x1862022294259200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768007 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767982.770208:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775767982.770217:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767982.770222:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775767982.770227:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00100000:3.0:1775767982.770230:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207889f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294259200:192.168.202.156@tcp:250:kworker.0 00000100:00000001:1.0:1775767982.770232:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775767982.770238:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:3.0:1775767982.770241:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:1.0:1775767982.770242:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:1.0:1775767982.770246:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:3.0:1775767982.770246:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000020:00000040:1.0:1775767982.770250:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000100:00000040:3.0:1775767982.770254:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207889f80 x1862022294259200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768007 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1775767982.770255:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:1.0:1775767982.770261:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:1.0:1775767982.770271:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767982.770273:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 00000100:00000001:1.0:1775767982.770276:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1775767982.770278:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000040:1.0:1775767982.770281:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000010:3.0:1775767982.770283:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b0000. 00000100:00000001:1.0:1775767982.770285:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 02000000:00000001:3.0:1775767982.770288:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000040:1.0:1775767982.770289:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000020:00000001:3.0:1775767982.770291:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000100:00080000:1.0:1775767982.770295:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000020:00000040:3.0:1775767982.770295:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:3.0:1775767982.770299:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1775767982.770303:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767982.770303:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543c800. 02000000:00000001:1.0:1775767982.770309:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767982.770309:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767982.770312:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767982.770313:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207889f80. 02000000:00000001:1.0:1775767982.770314:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767982.770316:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767982.770319:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1775767982.770320:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767982.770322:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767982.770325:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775767982.770328:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000100:00000001:3.0:1775767982.770329:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767982.770332:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1775767982.770333:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767982.770334:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767982.770337:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767982.770338:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767982.770339:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000010:1.0:1775767982.770346:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207889880. 00000100:00000040:3.0:1775767982.770346:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff99420788b100 x1862022294259328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1775767982.770352:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00000100:00000001:3.0:1775767982.770359:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 02000000:00000001:1.0:1775767982.770360:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1775767982.770361:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767982.770362:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767982.770363:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1775767982.770367:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767982.770367:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775767982.770371:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767982.770371:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767982.770376:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767982.770376:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff99420788b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294259328:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767982.770383:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000010:1.0:1775767982.770385:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543e800. 02000000:00000001:3.0:1775767982.770385:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767982.770386:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767982.770391:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1775767982.770392:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767982.770394:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b0000. 02000000:00000001:3.0:1775767982.770397:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767982.770402:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00100000:1.0:1775767982.770403:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207889880 x1862022294259584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:3.0:1775767982.770404:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767982.770409:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767982.770414:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294259328, portal 10 00000100:00000001:3.0:1775767982.770418:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767982.770420:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606130944 : -112966103420672 : ffff99420788b100) 00000100:00000001:1.0:1775767982.770423:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775767982.770428:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767982.770429:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff99420788b100 x1862022294259328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768007 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767982.770438:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775767982.770440:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f11856b8] to pc [ptlrpcd_rcv+-1] req@ffff994207889880 x1862022294259584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775767982.770443:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767982.770446:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767982.770451:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294259328, offset 0 00000400:00000200:3.0:1775767982.770459:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00000001:1.0:1775767982.770466:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767982.770466:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00080000:1.0:1775767982.770472:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:3.0:1775767982.770473:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00080000:1.0:1775767982.770480:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000400:00000200:3.0:1775767982.770481:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1775767982.770484:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:3.0:1775767982.770486:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000100:00080000:1.0:1775767982.770488:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:3.0:1775767982.770489:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767982.770495:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00000001:1.0:1775767982.770496:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:3.0:1775767982.770500:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225199:225199:256:4294967295] 192.168.202.156@tcp LPNI seq info [225195:225195:8:4294967295] 00000100:00080000:1.0:1775767982.770504:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5235 00000400:00000200:3.0:1775767982.770507:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767982.770513:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775767982.770518:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775767982.770523:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000400:00000200:3.0:1775767982.770524:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000100:00000040:1.0:1775767982.770532:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000800:00000200:3.0:1775767982.770533:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00000001:1.0:1775767982.770539:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000010:3.0:1775767982.770539:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000100:00000040:1.0:1775767982.770548:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000800:00000200:3.0:1775767982.770548:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767982.770553:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000100:00000001:1.0:1775767982.770554:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775767982.770559:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000800:00000200:3.1:1775767982.770559:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000040:1.0:1775767982.770562:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000800:00000200:3.1:1775767982.770563:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000020:00000001:1.0:1775767982.770565:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775767982.770569:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000800:00000100:3.0:1775767982.770569:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767982.770573:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000020:00000001:1.0:1775767982.770574:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000400:00000200:3.0:1775767982.770576:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000001:1.0:1775767982.770579:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000400:00000200:3.0:1775767982.770581:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767982.770586:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000100:00000040:1.0:1775767982.770588:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000010:3.0:1775767982.770589:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767982.770593:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1775767982.770594:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:3.0:1775767982.770599:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff99420788b100 x1862022294259328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768007 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775767982.770600:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775767982.770607:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775767982.770613:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00000001:3.0:1775767982.770614:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767982.770619:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff99420788b100 x1862022294259328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768007 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767982.770622:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775767982.770628:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767982.770633:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767982.770634:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1775767982.770636:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767982.770639:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775767982.770640:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1775767982.770642:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 02000000:00000001:1.0:1775767982.770644:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767982.770649:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775767982.770650:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775767982.770651:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299907603 00000008:00100000:1.0:1775767982.770654:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000100:00000001:3.0:1775767982.770655:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299908925 00000100:00000001:3.0:1775767982.770660:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299907603 00000008:00000001:1.0:1775767982.770661:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767982.770662:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299908925 00000100:00000001:1.0:1775767982.770666:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767982.770667:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767982.770671:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767982.770673:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000010:1.0:1775767982.770674:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207888e00. 00000100:00100000:3.0:1775767982.770676:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767982/real 1775767982] req@ffff99420788b100 x1862022294259328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1775767982.770680:0:13471:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:1.0:1775767982.770687:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000001:3.0:1775767982.770688:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:1.0:1775767982.770691:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1775767982.770691:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767982.770693:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000100:00000001:1.0:1775767982.770696:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:3.0:1775767982.770697:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 02000000:00000001:1.0:1775767982.770699:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:3.0:1775767982.770700:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 02000000:00000001:1.0:1775767982.770703:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767982.770703:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767982.770708:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff99420788b100 x1862022294259328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1775767982.770710:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543c800. 00000100:00000001:1.0:1775767982.770718:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1775767982.770718:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff99420788b100 x1862022294259328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767982.770727:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00100000:1.0:1775767982.770729:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207888e00 x1862022294259712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767982.770729:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767982.770733:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767982.770735:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767982.770743:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff99420788b100 x1862022294259328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775767982.770752:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767982.770753:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767982.770758:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767982.770758:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff99420788b100 x1862022294259328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768007 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775767982.770767:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001812f966] to pc [ptlrpcd_rcv+-1] req@ffff994207888e00 x1862022294259712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767982.770768:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767982.770773:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767982.770777:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767982.770779:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767982.770783:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767982.770787:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767982.770792:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00000001:1.0:1775767982.770793:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767982.770797:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000040:1.0:1775767982.770800:0:13471:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5245) 00000100:00000001:3.0:1775767982.770800:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767982.770807:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff99420788b100 x1862022294259328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768007 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767982.770813:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff99420788b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294259328:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767982.770816:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767982.770819:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207888700 x1862022294259456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767982.770823:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767982.770824:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767982.770826:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767982.770827:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767982.770828:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767982.770831:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207888700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294259456:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767982.770833:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767982.770835:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767982.770836:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767982.770837:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767982.770839:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b2c00. 02000000:00000001:3.0:1775767982.770841:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767982.770843:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767982.770845:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767982.770847:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767982.770849:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294259456, portal 10 00000100:00000001:3.0:1775767982.770850:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767982.770851:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606120192 : -112966103431424 : ffff994207888700) 00000100:00000040:3.0:1775767982.770855:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207888700 x1862022294259456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768007 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767982.770860:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767982.770862:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767982.770863:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767982.770864:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294259456, offset 0 00000400:00000200:3.0:1775767982.770867:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767982.770869:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767982.770871:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767982.770873:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767982.770875:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767982.770877:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767982.770878:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767982.770880:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225200:225200:256:4294967295] 192.168.202.156@tcp LPNI seq info [225196:225196:8:4294967295] 00000400:00000200:3.0:1775767982.770883:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767982.770887:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767982.770890:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767982.770892:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767982.770894:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767982.770896:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775767982.770898:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767982.770901:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000100:3.0:1775767982.770903:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767982.770905:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767982.770907:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767982.770909:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767982.770911:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767982.770912:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767982.770913:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767982.770915:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207888700 x1862022294259456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768007 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767982.770924:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767982.770926:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207888700 x1862022294259456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768007 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767982.770930:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767982.770931:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767982.770933:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767982.770935:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:3.0:1775767982.770937:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767982.770938:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299907595 00000100:00000001:3.0:1775767982.770939:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299908925 00000100:00000001:3.0:1775767982.770941:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299907595 00000100:00000001:3.0:1775767982.770942:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299908925 00000100:00000001:3.0:1775767982.770944:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767982.770945:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767982.770947:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767982.770950:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767982/real 1775767982] req@ffff994207888700 x1862022294259456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767982.770959:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767982.770960:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767982.770962:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767982.770964:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767982.770966:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767982.770969:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767982.770972:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207888700 x1862022294259456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767982.770981:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207888700 x1862022294259456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767982.770989:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767982.770990:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767982.770992:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767982.770994:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767982.770997:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207888700 x1862022294259456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767982.771002:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767982.771004:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207888700 x1862022294259456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768007 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767982.771008:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767982.771010:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767982.771011:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767982.771012:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767982.771013:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767982.771015:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767982.771016:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767982.771018:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767982.771020:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767982.771022:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207888700 x1862022294259456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768007 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767982.771027:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207888700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294259456:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767982.771030:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767982.771031:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767982.771033:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff99420788b100 x1862022294259328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768007 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767982.771037:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767982.771038:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767982.771039:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b0000. 02000000:00000001:3.0:1775767982.771041:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767982.771041:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767982.771043:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:3.0:1775767982.771044:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767982.771045:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543c400. 02000000:00000001:3.0:1775767982.771047:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767982.771048:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767982.771049:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff99420788b100. 00000100:00000001:3.0:1775767982.771051:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767982.771052:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767982.771053:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767982.771055:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207888700 x1862022294259456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768007 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767982.771074:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767982.771075:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767982.771076:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b2c00. 02000000:00000001:3.0:1775767982.771078:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767982.771079:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767982.771080:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:3.0:1775767982.771081:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767982.771082:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543f800. 02000000:00000001:3.0:1775767982.771084:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767982.771085:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767982.771085:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207888700. 00000100:00000001:3.0:1775767982.771087:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767982.771088:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767982.771089:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767982.771091:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767982.771092:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767982.771093:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767982.771094:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767982.771095:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767982.771098:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207889880 x1862022294259584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767982.771103:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767982.771104:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767982.771105:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767982.771106:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767982.771108:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767982.771110:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207889880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294259584:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767982.771112:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767982.771114:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767982.771115:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767982.771116:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767982.771118:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b2c00. 02000000:00000001:3.0:1775767982.771120:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767982.771122:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767982.771124:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767982.771126:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767982.771129:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294259584, portal 4 00000100:00000001:3.0:1775767982.771130:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767982.771131:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606124672 : -112966103426944 : ffff994207889880) 00000100:00000040:3.0:1775767982.771134:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207889880 x1862022294259584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768007 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767982.771139:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767982.771140:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767982.771142:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767982.771144:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294259584, offset 0 00000400:00000200:3.0:1775767982.771146:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767982.771149:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767982.771151:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767982.771153:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767982.771170:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767982.771172:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767982.771174:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767982.771176:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225201:225201:256:4294967295] 192.168.202.156@tcp LPNI seq info [225197:225197:8:4294967295] 00000400:00000200:3.0:1775767982.771179:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767982.771184:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767982.771187:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767982.771189:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767982.771192:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767982.771194:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775767982.771197:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767982.771199:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000100:3.0:1775767982.771201:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767982.771203:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767982.771205:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767982.771207:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767982.771209:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767982.771211:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767982.771213:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767982.771217:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207889880 x1862022294259584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768007 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767982.771225:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767982.771228:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207889880 x1862022294259584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768007 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767982.771235:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767982.771236:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767982.771240:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767982.771242:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:3.0:1775767982.771246:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767982.771248:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299907619 00000100:00000001:3.0:1775767982.771251:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299908925 00000100:00000001:3.0:1775767982.771253:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299907619 00000100:00000001:3.0:1775767982.771255:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299908925 00000100:00000001:3.0:1775767982.771257:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767982.771259:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767982.771261:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767982.771265:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767982/real 1775767982] req@ffff994207889880 x1862022294259584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767982.771280:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767982.771281:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767982.771283:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767982.771286:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767982.771288:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767982.771290:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767982.771293:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207889880 x1862022294259584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767982.771302:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207889880 x1862022294259584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767982.771309:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767982.771311:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767982.771313:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767982.771314:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767982.771318:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207889880 x1862022294259584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767982.771326:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767982.771330:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207889880 x1862022294259584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768007 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767982.771338:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767982.771340:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767982.771342:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767982.771343:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767982.771345:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767982.771349:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767982.771351:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767982.771355:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767982.771358:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767982.771363:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207889880 x1862022294259584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768007 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767982.771371:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207889880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294259584:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767982.771375:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775767982.771378:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207888e00 x1862022294259712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767982.771385:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767982.771386:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767982.771389:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775767982.771391:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767982.771393:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767982.771396:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207888e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294259712:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767982.771399:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775767982.771401:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767982.771403:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767982.771405:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775767982.771408:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b0000. 02000000:00000001:3.0:1775767982.771410:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767982.771413:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775767982.771416:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775767982.771419:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775767982.771423:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294259712, portal 4 00000100:00000001:3.0:1775767982.771424:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775767982.771425:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606121984 : -112966103429632 : ffff994207888e00) 00000100:00000040:3.0:1775767982.771429:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207888e00 x1862022294259712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768007 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767982.771434:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775767982.771435:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775767982.771437:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775767982.771439:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294259712, offset 0 00000400:00000200:3.0:1775767982.771442:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775767982.771445:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767982.771447:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775767982.771449:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775767982.771451:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775767982.771453:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775767982.771454:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775767982.771457:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225202:225202:256:4294967295] 192.168.202.156@tcp LPNI seq info [225198:225198:8:4294967295] 00000400:00000200:3.0:1775767982.771459:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767982.771464:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775767982.771466:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767982.771468:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775767982.771471:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775767982.771473:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775767982.771475:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775767982.771477:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000100:3.0:1775767982.771480:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775767982.771481:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775767982.771483:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775767982.771485:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775767982.771488:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775767982.771489:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775767982.771490:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775767982.771493:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207888e00 x1862022294259712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768007 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767982.771498:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767982.771500:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207888e00 x1862022294259712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768007 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767982.771504:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767982.771505:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775767982.771507:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767982.771509:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:3.0:1775767982.771511:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767982.771512:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299907622 00000100:00000001:3.0:1775767982.771514:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299908925 00000100:00000001:3.0:1775767982.771515:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299907622 00000100:00000001:3.0:1775767982.771517:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299908925 00000100:00000001:3.0:1775767982.771518:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775767982.771519:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767982.771521:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775767982.771523:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767982/real 1775767982] req@ffff994207888e00 x1862022294259712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775767982.771527:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775767982.771528:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775767982.771529:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775767982.771531:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775767982.771532:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775767982.771533:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775767982.771535:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207888e00 x1862022294259712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775767982.771540:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207888e00 x1862022294259712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767982.771543:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775767982.771544:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767982.771545:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767982.771546:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775767982.771548:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207888e00 x1862022294259712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767982.771553:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767982.771555:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207888e00 x1862022294259712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768007 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767982.771559:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767982.771562:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767982.771563:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767982.771563:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767982.771565:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775767982.771567:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775767982.771569:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767982.771571:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775767982.771572:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775767982.771574:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207888e00 x1862022294259712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768007 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767982.771579:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207888e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294259712:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775767982.771582:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767982.771583:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767982.771585:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207889880 x1862022294259584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768007 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767982.771589:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767982.771590:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767982.771591:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b2c00. 02000000:00000001:3.0:1775767982.771592:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767982.771593:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767982.771594:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:3.0:1775767982.771595:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767982.771596:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543e800. 02000000:00000001:3.0:1775767982.771598:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767982.771598:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767982.771599:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207889880. 00000100:00000001:3.0:1775767982.771601:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767982.771602:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767982.771603:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775767982.771605:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207888e00 x1862022294259712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768007 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767982.771613:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775767982.771614:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775767982.771615:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b0000. 02000000:00000001:3.0:1775767982.771616:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775767982.771617:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775767982.771617:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:3.0:1775767982.771619:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775767982.771619:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543c800. 02000000:00000001:3.0:1775767982.771621:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775767982.771621:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775767982.771622:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207888e00. 00000100:00000001:3.0:1775767982.771623:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775767982.771624:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767982.771625:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767982.771628:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767982.771628:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767982.771630:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775767982.771630:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767982.833266:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767982.833272:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767982.833278:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767982.833280:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767982.833284:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767982.833286:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767982.833289:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767982.833291:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767982.833295:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767982.833296:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767982.833323:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767982.833326:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767982.833329:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767982.897241:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767982.897249:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767982.897251:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767982.897257:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767982.897261:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767982.897266:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767982.897268:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767982.897271:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767982.897274:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767983.793236:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767983.793238:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767983.793240:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767983.793244:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767983.793245:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767983.793245:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767983.793246:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767983.793250:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767983.793250:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767983.793252:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:0.0:1775767983.793258:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775767983.793258:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1775767983.793258:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767983.793262:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767983.793265:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767983.793265:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767983.793265:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767983.793269:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767983.793271:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767983.793276:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767983.793279:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767983.793282:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767983.793285:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767983.793288:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767983.793289:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767983.793293:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767983.793296:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767983.857252:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767983.857257:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767983.857261:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767983.857263:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767983.857266:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767983.857267:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767983.857269:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767983.857270:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767983.857272:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767983.857273:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767983.857289:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767983.857291:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767983.857293:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767983.921218:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767983.921223:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767983.921225:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767983.921229:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767983.921232:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767983.921235:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767983.921237:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767983.921239:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767983.921241:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767984.113256:0:14953:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4299908925, at 4299908928 00000100:00000040:1.0:1775767984.113257:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299908925, at 4299908928 00000100:00000010:0.0:1775767984.113271:0:14953:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207981f60. 00000100:00000010:1.0:1775767984.113274:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207981900. 00000100:00000001:1.0:1775767984.817264:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767984.817274:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767984.817274:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767984.817278:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767984.817279:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767984.817280:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767984.817288:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767984.817290:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:0.0:1775767984.817291:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:3.0:1775767984.817296:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775767984.817298:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767984.817302:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767984.817302:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:0.0:1775767984.817305:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767984.817311:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767984.817312:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767984.817313:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767984.817315:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767984.817317:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767984.817320:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767984.817333:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767984.817334:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767984.817336:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767984.817340:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767984.817342:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767984.817349:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767984.817354:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767984.881291:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767984.881299:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767984.881307:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767984.881309:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767984.881330:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767984.881331:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767984.881334:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767984.881335:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767984.881338:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767984.881340:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767984.881360:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767984.881363:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767984.881365:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767984.945271:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767984.945278:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767984.945280:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767984.945306:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767984.945311:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767984.945317:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767984.945319:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767984.945323:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767984.945326:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767985.841225:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767985.841231:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767985.841232:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767985.841233:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767985.841236:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767985.841239:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767985.841239:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:0.0:1775767985.841243:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775767985.841243:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767985.841247:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767985.841248:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767985.841250:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767985.841251:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767985.841254:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767985.841258:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767985.841260:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767985.841263:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767985.841266:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767985.841266:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767985.841276:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767985.841277:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767985.841283:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775767985.841286:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767985.841290:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767985.841292:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767985.841295:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767985.841297:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767985.905239:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767985.905245:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767985.905249:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767985.905252:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775767985.905253:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775767985.905257:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767985.905259:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775767985.905262:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767985.905263:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767985.905267:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767985.905271:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767985.905272:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775767985.905273:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775767985.905274:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767985.905275:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775767985.905276:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767985.905277:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767985.905284:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767985.905289:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767985.905290:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775767985.905291:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775767985.905292:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767985.905294:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775767985.905295:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767985.905296:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767985.905299:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767985.905301:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767985.905302:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775767985.905303:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775767985.905304:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767985.905305:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775767985.905306:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767985.905307:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767985.905308:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767985.905326:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767985.905328:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767985.905329:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767985.969218:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767985.969223:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767985.969225:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767985.969229:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767985.969232:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767985.969235:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767985.969237:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767985.969239:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767985.969241:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767986.865265:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767986.865266:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767986.865278:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767986.865278:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767986.865281:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767986.865287:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767986.865288:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767986.865288:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767986.865293:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767986.865305:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:0.0:1775767986.865306:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:1.0:1775767986.865306:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767986.865324:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767986.865325:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767986.865325:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767986.865328:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767986.865333:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767986.865334:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767986.865335:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767986.865340:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767986.865344:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767986.865345:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767986.865347:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767986.865350:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767986.865350:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767986.865354:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767986.865358:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767986.929310:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767986.929318:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767986.929324:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767986.929325:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767986.929330:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767986.929331:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767986.929334:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767986.929336:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767986.929340:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767986.929341:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767986.929370:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767986.929374:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767986.929377:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767986.993252:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767986.993259:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767986.993261:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767986.993267:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767986.993271:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767986.993275:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767986.993278:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767986.993280:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767986.993284:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767987.889218:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767987.889224:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767987.889226:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767987.889227:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:0.0:1775767987.889231:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775767987.889236:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767987.889236:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767987.889239:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767987.889239:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767987.889242:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:0.0:1775767987.889243:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767987.889245:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767987.889247:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775767987.889248:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775767987.889249:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767987.889250:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767987.889254:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767987.889257:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775767987.889258:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775767987.889261:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767987.889262:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767987.889265:0:8:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767987.889269:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775767987.889270:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767987.889274:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:3.0:1775767987.889276:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5240 00000100:00000001:2.0:1775767987.889277:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767987.889281:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767987.889285:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767987.889288:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1775767987.889288:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767987.889293:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767987.889295:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1775767987.889297:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:2.0:1775767987.889298:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767987.889301:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767987.889305:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767987.889307:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767987.889308:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767987.889310:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767987.889314:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767987.889315:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:3.0:1775767987.889317:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767987.889320:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:3.0:1775767987.889321:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:3.0:1775767987.889323:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:3.0:1775767987.889325:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767987.889327:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767987.889328:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767987.889329:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767987.889339:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:3.0:1775767987.889343:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767987.889345:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767987.889348:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767987.889352:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767987.889354:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767987.889356:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775767987.889359:0:8:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767987.889366:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8e00. 00000020:00000040:3.0:1775767987.889370:0:8:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:3.0:1775767987.889374:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767987.889376:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767987.889379:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767987.889382:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767987.889386:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767987.889394:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0000. 00000100:00000001:3.0:1775767987.889401:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767987.889408:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8e00 x1862022294259840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775767987.889416:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767987.889419:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767987.889423:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000864a6866] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8e00 x1862022294259840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767987.889442:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767987.889445:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767987.889449:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775767987.889458:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767987.889460:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767987.889464:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767987.889469:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5240 00000100:00000001:2.0:1775767987.889473:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:3.0:1775767987.889474:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767987.889477:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767987.889480:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000400:00000200:3.0:1775767987.889480:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767987.889485:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775767987.889489:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a8e00 x1862022294259840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775767987.889492:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767987.889497:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767987.889501:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767987.889506:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:2.0:1775767987.889507:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767987.889510:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767987.889510:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767987.889514:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 02000000:00000001:2.0:1775767987.889515:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:3.0:1775767987.889515:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000001:2.0:1775767987.889519:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:3.0:1775767987.889519:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 02000000:00000001:2.0:1775767987.889524:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767987.889524:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:3.0:1775767987.889527:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00100000:2.0:1775767987.889530:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294259840:192.168.202.156@tcp:250:kworker.0 00000100:00000040:3.0:1775767987.889533:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767987.889537:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775767987.889538:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000040:3.0:1775767987.889540:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000001:2.0:1775767987.889543:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1775767987.889543:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 02000000:00000001:2.0:1775767987.889546:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775767987.889546:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 02000000:00000001:2.0:1775767987.889550:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:3.0:1775767987.889551:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767987.889554:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775767987.889556:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1d000. 02000000:00000001:3.0:1775767987.889556:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775767987.889559:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767987.889560:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767987.889563:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767987.889567:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767987.889567:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767987.889571:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775767987.889575:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775767987.889578:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000100:00000001:3.0:1775767987.889579:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775767987.889587:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000010:3.0:1775767987.889587:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ab480. 00000020:00000040:3.0:1775767987.889593:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000100:00000200:2.0:1775767987.889596:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294259840, portal 25 02000000:00000001:3.0:1775767987.889597:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1775767987.889600:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1775767987.889601:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767987.889602:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640856064 : -112966068695552 : ffff9942099a8e00) 00000100:00000001:3.0:1775767987.889605:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767987.889608:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:2.0:1775767987.889610:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a8e00 x1862022294259840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768012 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1775767987.889612:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767987.889619:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2c00. 00000100:00000001:2.0:1775767987.889620:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767987.889624:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000100:00000001:3.0:1775767987.889626:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775767987.889628:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f880. 00000100:00000200:2.0:1775767987.889632:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294259840, offset 0 00000100:00100000:3.0:1775767987.889636:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ab480 x1862022294259968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1775767987.889642:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00000001:3.0:1775767987.889649:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767987.889651:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767987.889654:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000040:3.0:1775767987.889658:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000067a0ba74] to pc [ptlrpcd_rcv+-1] req@ffff9942099ab480 x1862022294259968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1775767987.889663:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767987.889672:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1775767987.889673:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767987.889676:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:2.0:1775767987.889677:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000100:00080000:3.0:1775767987.889681:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000200:2.0:1775767987.889684:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1775767987.889685:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767987.889688:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:2.0:1775767987.889690:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00000001:3.0:1775767987.889692:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767987.889696:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5240 00000400:00000200:2.0:1775767987.889698:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225203:225203:256:4294967295] 192.168.202.156@tcp LPNI seq info [225199:225199:8:4294967295] 00000400:00000200:2.0:1775767987.889702:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767987.889712:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767987.889716:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000400:00000200:3.0:1775767987.889719:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767987.889723:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767987.889727:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000800:00000200:2.0:1775767987.889728:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00000001:3.0:1775767987.889730:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767987.889733:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000800:00000010:2.0:1775767987.889734:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3a00. 00000100:00000001:3.0:1775767987.889736:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767987.889739:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767987.889740:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:3.0:1775767987.889741:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000800:00000200:2.0:1775767987.889744:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000040:3.0:1775767987.889744:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:3.0:1775767987.889747:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000800:00000200:2.0:1775767987.889749:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000020:00000001:3.0:1775767987.889750:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:3.0:1775767987.889756:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767987.889760:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:2.1:1775767987.889760:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000040:3.0:1775767987.889764:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000800:00000200:2.1:1775767987.889766:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000100:00000001:3.0:1775767987.889767:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767987.889771:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000800:00000100:2.0:1775767987.889776:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000100:00080000:3.0:1775767987.889776:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000800:00000010:2.0:1775767987.889781:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3a00. 00000100:00000001:3.0:1775767987.889781:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767987.889782:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767987.889786:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1775767987.889787:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000001:3.0:1775767987.889787:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767987.889791:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767987.889794:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000400:00000200:2.0:1775767987.889795:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000008:00000001:3.0:1775767987.889796:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767987.889800:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f880 00000400:00000010:2.0:1775767987.889802:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f880. 00000100:00000001:3.0:1775767987.889804:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767987.889805:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000010:3.0:1775767987.889809:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a9180. 00000100:00000200:2.0:1775767987.889810:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a8e00 x1862022294259840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768012 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1775767987.889816:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:3.0:1775767987.889822:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1775767987.889824:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1775767987.889824:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767987.889825:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767987.889828:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:2.0:1775767987.889829:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a8e00 x1862022294259840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768012 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1775767987.889830:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767987.889836:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2800. 00000100:00000001:2.0:1775767987.889840:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767987.889840:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767987.889843:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:3.0:1775767987.889847:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a9180 x1862022294260096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775767987.889849:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767987.889858:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767987.889863:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299914045 00000100:00000001:3.0:1775767987.889869:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000010:2.0:1775767987.889870:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d6de0. 00000100:00000001:3.0:1775767987.889871:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767987.889877:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bdc2586] to pc [ptlrpcd_rcv+-1] req@ffff9942099a9180 x1862022294260096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767987.889882:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767987.889884:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775767987.889886:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00080000:3.0:1775767987.889891:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1775767987.889893:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299914045 00000100:00000001:3.0:1775767987.889894:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767987.889897:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000010:2.0:1775767987.889898:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d6ba0. 00000100:00000001:2.0:1775767987.889903:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1775767987.889903:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775767987.889907:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1775767987.889909:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5240 00000100:00000001:2.0:1775767987.889910:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767987.889912:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767987.889916:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767987/real 1775767987] req@ffff9942099a8e00 x1862022294259840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1775767987.889926:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767987.889935:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000400:00000001:2.0:1775767987.889936:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767987.889939:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:3.0:1775767987.889939:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000400:00000010:2.0:1775767987.889941:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000100:00000040:3.0:1775767987.889945:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000200:2.0:1775767987.889946:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775767987.889949:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:3.0:1775767987.889950:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775767987.889952:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:3.0:1775767987.889955:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000200:2.0:1775767987.889958:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a8e00 x1862022294259840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767987.889960:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000200:2.0:1775767987.889967:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a8e00 x1862022294259840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775767987.889973:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767987.889976:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:3.0:1775767987.889980:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:2.0:1775767987.889982:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000040:3.0:1775767987.889983:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000100:00000001:2.0:1775767987.889985:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1775767987.889988:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:2.0:1775767987.889990:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:3.0:1775767987.889993:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:2.0:1775767987.889994:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775767987.890001:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00100000:2.0:1775767987.890003:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a8e00 x1862022294259840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767987.890006:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767987.890025:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767987.890030:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767987.890037:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00000001:2.0:1775767987.890038:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775767987.890041:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767987.890046:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767987.890047:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a8e00 x1862022294259840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768012 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1775767987.890049:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775767987.890054:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767987.890055:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:3.0:1775767987.890059:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767987.890060:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767987.890064:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767987.890064:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775767987.890066:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:3.0:1775767987.890068:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000100:00000001:2.0:1775767987.890069:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767987.890071:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767987.890075:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000008:00000001:3.0:1775767987.890075:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767987.890080:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775767987.890082:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767987.890085:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000010:3.0:1775767987.890086:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8700. 00000100:00000040:2.0:1775767987.890089:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a8e00 x1862022294259840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768012 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1775767987.890091:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:3.0:1775767987.890098:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767987.890101:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767987.890102:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294259840:192.168.202.156@tcp:250:kworker.0 00000100:00000001:3.0:1775767987.890105:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1775767987.890108:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1775767987.890108:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1775767987.890111:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1775767987.890112:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767987.890114:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a8e00 x1862022294259840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768012 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1775767987.890119:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0400. 00000100:00000001:2.0:1775767987.890121:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767987.890124:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1775767987.890125:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775767987.890126:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1d000. 02000000:00000001:2.0:1775767987.890133:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767987.890136:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000100:00100000:3.0:1775767987.890137:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8700 x1862022294260224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000040:2.0:1775767987.890138:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:2.0:1775767987.890143:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767987.890145:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b0000. 02000000:00000001:2.0:1775767987.890151:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767987.890178:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767987.890180:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a8e00. 00000100:00000001:2.0:1775767987.890183:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767987.890184:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767987.890186:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767987.890189:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767987.890192:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767987.890193:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767987.890196:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767987.890199:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1775767987.890201:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000faa25e3b] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8700 x1862022294260224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767987.890202:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767987.890205:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767987.890210:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ab480 x1862022294259968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767987.890228:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767987.890230:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767987.890230:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767987.890232:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767987.890235:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1775767987.890236:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:2.0:1775767987.890238:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767987.890243:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1775767987.890244:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ab480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294259968:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775767987.890248:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1775767987.890250:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00080000:3.0:1775767987.890251:0:8:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:2.0:1775767987.890252:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767987.890255:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767987.890256:0:8:0:(import.c:484:import_select_connection()) Process entered 02000000:00000001:2.0:1775767987.890257:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775767987.890261:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1d000. 00000100:00080000:3.0:1775767987.890263:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5240 02000000:00000001:2.0:1775767987.890264:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767987.890275:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767987.890275:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767987.890279:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000200:3.0:1775767987.890283:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000400:00000010:2.0:1775767987.890284:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00080000:3.0:1775767987.890289:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000200:2.0:1775767987.890290:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294259968, portal 10 00000100:00000040:3.0:1775767987.890295:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767987.890298:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775767987.890301:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000040:3.0:1775767987.890301:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767987.890303:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640865920 : -112966068685696 : ffff9942099ab480) 00000100:00000001:3.0:1775767987.890306:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767987.890310:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000040:2.0:1775767987.890311:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ab480 x1862022294259968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768012 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1775767987.890313:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:3.0:1775767987.890316:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767987.890320:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000100:00000001:2.0:1775767987.890322:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767987.890326:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000020:00000001:3.0:1775767987.890326:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000400:00000010:2.0:1775767987.890330:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f880. 00000020:00000001:3.0:1775767987.890331:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000200:2.0:1775767987.890333:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294259968, offset 0 00000100:00000040:3.0:1775767987.890339:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000200:2.0:1775767987.890340:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00000001:3.0:1775767987.890344:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1775767987.890346:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000040:3.0:1775767987.890349:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000200:2.0:1775767987.890351:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00000001:3.0:1775767987.890355:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000200:2.0:1775767987.890356:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000040:3.0:1775767987.890360:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000400:00000200:2.0:1775767987.890361:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767987.890366:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000100:00080000:3.0:1775767987.890368:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000400:00000200:2.0:1775767987.890371:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767987.890376:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225204:225204:256:4294967295] 192.168.202.156@tcp LPNI seq info [225200:225200:8:4294967295] 00000100:00000001:3.0:1775767987.890376:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767987.890381:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:2.0:1775767987.890384:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:3.0:1775767987.890386:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767987.890392:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767987.890393:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000100:00000001:3.0:1775767987.890398:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000800:00000200:2.0:1775767987.890401:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000008:00100000:3.0:1775767987.890402:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000800:00000010:2.0:1775767987.890405:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3a00. 00000008:00000001:3.0:1775767987.890407:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1775767987.890412:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:3.0:1775767987.890412:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1775767987.890418:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000100:00000010:3.0:1775767987.890420:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8380. 00000800:00000200:2.1:1775767987.890423:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000040:3.0:1775767987.890426:0:8:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00000800:00000200:2.1:1775767987.890426:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000100:2.0:1775767987.890433:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 02000000:00000001:3.0:1775767987.890434:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000800:00000010:2.0:1775767987.890435:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3a00. 02000000:00000001:3.0:1775767987.890438:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767987.890439:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:3.0:1775767987.890443:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:2.0:1775767987.890445:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 02000000:00000001:3.0:1775767987.890447:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:2.0:1775767987.890448:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f880 00000400:00000010:2.0:1775767987.890451:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f880. 02000000:00000001:3.0:1775767987.890452:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767987.890453:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767987.890459:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ab480 x1862022294259968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768012 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1775767987.890460:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b3400. 00000100:00000001:3.0:1775767987.890466:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767987.890473:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00100000:3.0:1775767987.890476:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8380 x1862022294260352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:2.0:1775767987.890481:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ab480 x1862022294259968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768012 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767987.890499:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775767987.890501:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767987.890503:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767987.890506:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:3.0:1775767987.890511:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055f5621a] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8380 x1862022294260352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767987.890512:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767987.890519:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000100:00000001:3.0:1775767987.890527:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767987.890528:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767987.890532:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299912723 00000100:00000040:3.0:1775767987.890534:0:8:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5250) 00000100:00000001:2.0:1775767987.890537:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299914045 00000100:00000001:2.0:1775767987.890541:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299912723 00000100:00000001:2.0:1775767987.890544:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299914045 00000100:00000001:2.0:1775767987.890547:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767987.890550:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767987.890552:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767987.890557:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767987/real 1775767987] req@ffff9942099ab480 x1862022294259968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775767987.890566:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767987.890567:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767987.890570:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775767987.890572:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775767987.890574:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775767987.890577:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767987.890580:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ab480 x1862022294259968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767987.890588:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ab480 x1862022294259968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767987.890594:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767987.890595:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767987.890596:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767987.890597:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767987.890600:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ab480 x1862022294259968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767987.890604:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767987.890606:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ab480 x1862022294259968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768012 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767987.890611:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767987.890613:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767987.890614:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767987.890615:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767987.890616:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767987.890617:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767987.890619:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767987.890621:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767987.890622:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767987.890625:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ab480 x1862022294259968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768012 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767987.890629:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ab480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294259968:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775767987.890633:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767987.890635:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a9180 x1862022294260096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767987.890640:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767987.890641:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767987.890642:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767987.890643:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767987.890644:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767987.890646:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294260096:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775767987.890648:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767987.890649:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767987.890650:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767987.890652:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775767987.890654:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f800. 02000000:00000001:2.0:1775767987.890656:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767987.890658:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767987.890660:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775767987.890662:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775767987.890664:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294260096, portal 10 00000100:00000001:2.0:1775767987.890666:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767987.890667:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640856960 : -112966068694656 : ffff9942099a9180) 00000100:00000040:2.0:1775767987.890671:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a9180 x1862022294260096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768012 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767987.890675:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767987.890677:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767987.890679:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f880. 00000100:00000200:2.0:1775767987.890680:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294260096, offset 0 00000400:00000200:2.0:1775767987.890683:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767987.890685:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767987.890688:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767987.890690:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767987.890692:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767987.890694:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767987.890696:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767987.890698:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225205:225205:256:4294967295] 192.168.202.156@tcp LPNI seq info [225201:225201:8:4294967295] 00000400:00000200:2.0:1775767987.890701:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767987.890705:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767987.890708:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767987.890710:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3a00. 00000800:00000200:2.0:1775767987.890712:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775767987.890714:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775767987.890718:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767987.890720:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000100:2.0:1775767987.890722:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767987.890724:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3a00. 00000400:00000200:2.0:1775767987.890726:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775767987.890728:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775767987.890730:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f880 00000400:00000010:2.0:1775767987.890732:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f880. 00000100:00000001:2.0:1775767987.890733:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767987.890736:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a9180 x1862022294260096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768012 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767987.890741:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767987.890743:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a9180 x1862022294260096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768012 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767987.890747:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767987.890748:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775767987.890750:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767987.890752:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:2.0:1775767987.890754:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767987.890755:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299912715 00000100:00000001:2.0:1775767987.890757:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299914045 00000100:00000001:2.0:1775767987.890759:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299912715 00000100:00000001:2.0:1775767987.890760:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299914045 00000100:00000001:2.0:1775767987.890762:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767987.890764:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767987.890765:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767987.890767:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767987/real 1775767987] req@ffff9942099a9180 x1862022294260096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775767987.890772:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767987.890772:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767987.890773:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775767987.890775:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775767987.890775:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775767987.890777:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767987.890778:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a9180 x1862022294260096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767987.890784:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a9180 x1862022294260096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767987.890788:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767987.890789:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767987.890790:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767987.890791:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767987.890793:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a9180 x1862022294260096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767987.890797:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767987.890799:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a9180 x1862022294260096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768012 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767987.890803:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767987.890804:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767987.890805:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767987.890806:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767987.890807:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767987.890810:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767987.890812:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767987.890814:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767987.890815:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767987.890818:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a9180 x1862022294260096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768012 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767987.890822:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294260096:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775767987.890824:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767987.890826:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767987.890828:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ab480 x1862022294259968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768012 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767987.890832:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767987.890833:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767987.890834:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1d000. 02000000:00000001:2.0:1775767987.890836:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767987.890837:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767987.890838:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:2.0:1775767987.890839:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767987.890840:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b2c00. 02000000:00000001:2.0:1775767987.890842:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767987.890843:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767987.890844:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ab480. 00000100:00000001:2.0:1775767987.890846:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767987.890847:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767987.890848:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767987.890850:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a9180 x1862022294260096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768012 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767987.890854:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767987.890855:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767987.890856:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f800. 02000000:00000001:2.0:1775767987.890858:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767987.890859:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767987.890860:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:2.0:1775767987.890861:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767987.890862:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b2800. 02000000:00000001:2.0:1775767987.890864:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767987.890864:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767987.890865:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a9180. 00000100:00000001:2.0:1775767987.890866:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767987.890867:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767987.890868:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767987.890870:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767987.890871:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767987.890873:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775767987.890873:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767987.890874:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767987.890876:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a8700 x1862022294260224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767987.890880:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767987.890881:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767987.890882:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767987.890884:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767987.890885:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767987.890887:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294260224:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767987.890889:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767987.890890:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767987.890891:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767987.890892:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775767987.890894:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f800. 02000000:00000001:2.0:1775767987.890895:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767987.890897:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767987.890899:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775767987.890901:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775767987.890903:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294260224, portal 4 00000100:00000001:2.0:1775767987.890904:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767987.890905:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640854272 : -112966068697344 : ffff9942099a8700) 00000100:00000040:2.0:1775767987.890909:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a8700 x1862022294260224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768012 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767987.890913:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767987.890915:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767987.890916:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f880. 00000100:00000200:2.0:1775767987.890917:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294260224, offset 0 00000400:00000200:2.0:1775767987.890919:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767987.890922:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767987.890924:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767987.890926:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767987.890927:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767987.890929:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767987.890930:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767987.890933:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225206:225206:256:4294967295] 192.168.202.156@tcp LPNI seq info [225202:225202:8:4294967295] 00000400:00000200:2.0:1775767987.890935:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767987.890939:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767987.890942:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767987.890943:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3a00. 00000800:00000200:2.0:1775767987.890946:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775767987.890948:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775767987.890950:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767987.890952:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000100:2.0:1775767987.890954:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767987.890956:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3a00. 00000400:00000200:2.0:1775767987.890958:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775767987.890960:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775767987.890962:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f880 00000400:00000010:2.0:1775767987.890963:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f880. 00000100:00000001:2.0:1775767987.890964:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767987.890966:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a8700 x1862022294260224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768012 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767987.890973:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767987.890977:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a8700 x1862022294260224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768012 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767987.890984:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767987.890986:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775767987.890989:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767987.890993:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:2.0:1775767987.890997:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767987.890999:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299912738 00000100:00000001:2.0:1775767987.891002:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299914045 00000100:00000001:2.0:1775767987.891004:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299912738 00000100:00000001:2.0:1775767987.891007:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299914045 00000100:00000001:2.0:1775767987.891009:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767987.891027:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767987.891029:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767987.891033:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767987/real 1775767987] req@ffff9942099a8700 x1862022294260224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775767987.891039:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767987.891040:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767987.891042:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775767987.891044:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775767987.891045:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775767987.891046:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767987.891048:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a8700 x1862022294260224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767987.891056:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a8700 x1862022294260224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767987.891063:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767987.891065:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767987.891066:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767987.891068:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767987.891072:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a8700 x1862022294260224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767987.891080:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767987.891083:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a8700 x1862022294260224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768012 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767987.891092:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767987.891094:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767987.891097:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767987.891098:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767987.891100:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767987.891104:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767987.891106:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767987.891110:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767987.891112:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767987.891117:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a8700 x1862022294260224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768012 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767987.891126:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294260224:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767987.891130:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767987.891134:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a8380 x1862022294260352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767987.891142:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767987.891143:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767987.891146:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767987.891148:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767987.891150:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767987.891153:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294260352:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767987.891173:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767987.891176:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767987.891177:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767987.891178:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775767987.891181:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1d000. 02000000:00000001:2.0:1775767987.891182:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767987.891185:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767987.891186:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775767987.891189:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775767987.891191:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294260352, portal 4 00000100:00000001:2.0:1775767987.891193:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767987.891194:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640853376 : -112966068698240 : ffff9942099a8380) 00000100:00000040:2.0:1775767987.891199:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a8380 x1862022294260352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768012 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767987.891203:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767987.891205:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767987.891206:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f880. 00000100:00000200:2.0:1775767987.891209:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294260352, offset 0 00000400:00000200:2.0:1775767987.891211:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767987.891214:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767987.891216:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767987.891218:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767987.891220:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767987.891222:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767987.891223:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767987.891226:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225207:225207:256:4294967295] 192.168.202.156@tcp LPNI seq info [225203:225203:8:4294967295] 00000400:00000200:2.0:1775767987.891229:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767987.891233:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767987.891236:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767987.891238:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3a00. 00000800:00000200:2.0:1775767987.891241:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775767987.891243:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775767987.891246:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767987.891248:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000100:2.0:1775767987.891251:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767987.891252:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3a00. 00000400:00000200:2.0:1775767987.891254:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775767987.891257:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775767987.891259:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f880 00000400:00000010:2.0:1775767987.891260:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f880. 00000100:00000001:2.0:1775767987.891261:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767987.891264:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a8380 x1862022294260352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768012 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767987.891269:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767987.891271:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a8380 x1862022294260352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768012 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767987.891275:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767987.891276:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775767987.891278:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767987.891280:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:2.0:1775767987.891282:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767987.891283:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299912742 00000100:00000001:2.0:1775767987.891285:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299914045 00000100:00000001:2.0:1775767987.891286:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299912742 00000100:00000001:2.0:1775767987.891288:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299914045 00000100:00000001:2.0:1775767987.891289:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767987.891291:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767987.891292:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767987.891294:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767987/real 1775767987] req@ffff9942099a8380 x1862022294260352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775767987.891299:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767987.891299:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767987.891300:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775767987.891302:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775767987.891303:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775767987.891305:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767987.891307:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a8380 x1862022294260352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767987.891313:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a8380 x1862022294260352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767987.891316:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767987.891317:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767987.891318:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767987.891319:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767987.891321:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a8380 x1862022294260352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767987.891325:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767987.891327:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a8380 x1862022294260352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768012 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767987.891331:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767987.891333:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767987.891335:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767987.891337:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767987.891338:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767987.891342:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767987.891345:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767987.891349:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767987.891351:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767987.891355:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a8380 x1862022294260352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768012 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767987.891363:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294260352:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767987.891367:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767987.891369:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767987.891372:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a8700 x1862022294260224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768012 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767987.891379:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767987.891381:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767987.891382:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f800. 02000000:00000001:2.0:1775767987.891385:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767987.891387:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767987.891388:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:2.0:1775767987.891391:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767987.891392:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b0400. 02000000:00000001:2.0:1775767987.891395:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767987.891397:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767987.891398:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a8700. 00000100:00000001:2.0:1775767987.891401:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767987.891402:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767987.891404:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767987.891408:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a8380 x1862022294260352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768012 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767987.891415:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767987.891417:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767987.891418:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1d000. 02000000:00000001:2.0:1775767987.891421:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767987.891422:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767987.891423:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:2.0:1775767987.891425:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767987.891427:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b3400. 02000000:00000001:2.0:1775767987.891429:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767987.891431:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767987.891432:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a8380. 00000100:00000001:2.0:1775767987.891434:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767987.891436:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767987.891438:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767987.891442:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767987.891443:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767987.891446:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775767987.891447:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767987.953259:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767987.953263:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767987.953268:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767987.953270:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767987.953272:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767987.953273:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767987.953275:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767987.953276:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767987.953279:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767987.953280:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767987.953295:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767987.953297:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767987.953300:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767988.017280:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767988.017287:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767988.017289:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767988.017295:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767988.017298:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767988.017303:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767988.017305:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767988.017308:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767988.017311:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767988.913288:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767988.913288:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767988.913291:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767988.913297:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767988.913297:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767988.913301:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767988.913302:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767988.913302:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767988.913309:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767988.913313:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:1.0:1775767988.913313:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:2.0:1775767988.913317:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767988.913319:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767988.913320:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767988.913328:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767988.913330:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767988.913332:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767988.913334:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767988.913337:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767988.913341:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767988.913345:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767988.913346:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767988.913349:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767988.913353:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767988.913356:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767988.913360:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767988.913365:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767988.977281:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767988.977288:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767988.977293:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767988.977294:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767988.977298:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767988.977300:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767988.977302:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767988.977303:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767988.977306:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767988.977307:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767988.977326:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767988.977328:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767988.977330:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767989.041266:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767989.041274:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767989.041276:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767989.041282:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767989.041286:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767989.041290:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767989.041292:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767989.041294:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767989.041296:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767989.233237:0:14953:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4299914045, at 4299914048 00000100:00000010:0.0:1775767989.233257:0:14953:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d6de0. 00000100:00000040:1.0:1775767989.233333:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299914045, at 4299914048 00000100:00000010:1.0:1775767989.233340:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d6ba0. 00000100:00000001:0.0:1775767989.937254:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767989.937262:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767989.937263:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767989.937270:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775767989.937275:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767989.937280:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767989.937283:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767989.937286:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767989.937288:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767989.937305:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767989.937306:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767989.937312:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767989.937313:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767989.937317:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767989.937321:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767989.937323:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:2.0:1775767989.937330:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775767989.937331:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767989.937337:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767989.937338:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767989.937344:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767989.937345:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767989.937349:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767989.937353:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767989.937355:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767989.937357:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767989.937360:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767990.001283:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767990.001289:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767990.001294:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767990.001296:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767990.001299:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767990.001300:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767990.001303:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767990.001304:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767990.001308:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767990.001309:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767990.001328:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767990.001330:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767990.001333:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767990.065273:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767990.065280:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767990.065282:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767990.065287:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767990.065291:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767990.065295:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767990.065297:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767990.065300:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767990.065303:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767990.961299:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767990.961304:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767990.961306:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767990.961310:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767990.961311:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767990.961314:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767990.961317:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:2.0:1775767990.961320:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775767990.961320:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767990.961322:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767990.961327:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767990.961329:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767990.961330:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1775767990.961332:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775767990.961335:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767990.961337:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767990.961337:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767990.961339:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767990.961341:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767990.961342:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767990.961348:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767990.961349:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767990.961353:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767990.961357:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767990.961358:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767990.961360:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767990.961364:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767991.025258:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767991.025269:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775767991.025271:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775767991.025275:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767991.025282:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775767991.025284:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767991.025287:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767991.025289:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767991.025293:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767991.025295:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767991.025299:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767991.025301:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767991.025304:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767991.025305:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767991.025309:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767991.025310:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767991.025333:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767991.025336:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767991.025339:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767991.089256:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767991.089265:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767991.089267:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767991.089272:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767991.089276:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767991.089282:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767991.089284:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767991.089287:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767991.089289:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767991.985256:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775767991.985256:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767991.985263:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767991.985265:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767991.985265:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767991.985268:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767991.985268:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775767991.985269:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767991.985271:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767991.985275:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:1.0:1775767991.985276:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775767991.985276:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767991.985283:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767991.985284:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767991.985286:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767991.985287:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767991.985289:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767991.985293:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767991.985295:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767991.985297:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767991.985298:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767991.985298:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767991.985302:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767991.985305:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767991.985313:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767991.985316:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767991.985324:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767992.049304:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767992.049313:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767992.049322:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767992.049323:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767992.049328:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767992.049329:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767992.049333:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767992.049335:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767992.049339:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767992.049340:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767992.049368:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767992.049371:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767992.049373:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767992.113299:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767992.113307:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767992.113309:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767992.113314:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767992.113318:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767992.113322:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767992.113324:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767992.113327:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767992.113330:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767993.009290:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767993.009294:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767993.009300:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767993.009301:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767993.009303:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767993.009303:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:3.0:1775767993.009308:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775767993.009309:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767993.009310:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:1.0:1775767993.009313:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775767993.009318:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767993.009320:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00080000:3.0:1775767993.009322:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:0.0:1775767993.009325:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767993.009328:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775767993.009330:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1775767993.009331:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767993.009334:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767993.009338:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775767993.009340:0:8:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775767993.009341:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767993.009341:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767993.009344:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767993.009346:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775767993.009348:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767993.009356:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767993.009357:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:3.0:1775767993.009357:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5245 00000100:00000001:0.0:1775767993.009365:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767993.009372:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767993.009373:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000400:00000200:3.0:1775767993.009375:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767993.009378:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767993.009387:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767993.009391:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767993.009395:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767993.009400:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767993.009404:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767993.009409:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000080:00000001:1.0:1775767993.009412:0:13471:0:(rw.c:1481:ll_writepages()) Process entered 00000100:00000001:3.0:1775767993.009412:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767993.009415:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000080:00000001:1.0:1775767993.009417:0:13471:0:(file.c:5250:cl_sync_file_range()) Process entered 00000020:00000040:3.0:1775767993.009418:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:1.0:1775767993.009422:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000001:3.0:1775767993.009423:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767993.009427:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:3.0:1775767993.009428:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:3.0:1775767993.009431:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000010:1.0:1775767993.009434:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994206aba240. 00000100:00000040:3.0:1775767993.009438:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000010:1.0:1775767993.009441:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994208a8f600. 00000100:00000001:3.0:1775767993.009441:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767993.009444:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767993.009447:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000010:1.0:1775767993.009449:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994208a8ec00. 00000020:00000040:3.0:1775767993.009449:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000020:00000010:1.0:1775767993.009455:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207cba9c0. 00000100:00080000:3.0:1775767993.009455:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767993.009457:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767993.009461:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000008:00000010:1.0:1775767993.009463:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207e02ca0. 02000000:00000001:3.0:1775767993.009464:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767993.009468:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767993.009470:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00020000:00000010:1.0:1775767993.009471:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff9942031b3480. 00000100:00000001:3.0:1775767993.009472:0:8:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:1.0:1775767993.009476:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942054ab170. 00000100:00000010:3.0:1775767993.009481:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8380. 00000080:00000010:1.0:1775767993.009484:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99422030bf18. 00000020:00000040:3.0:1775767993.009486:0:8:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 00000080:00000010:1.0:1775767993.009488:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff994205751da8. 02000000:00000001:3.0:1775767993.009491:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000020:00000010:1.0:1775767993.009493:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994208a8e000. 02000000:00000001:3.0:1775767993.009493:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767993.009497:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000008:00000010:1.0:1775767993.009498:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff994210709ce0. 02000000:00000001:3.0:1775767993.009501:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00020000:00000010:1.0:1775767993.009503:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994205743180. 00008000:00000010:1.0:1775767993.009507:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994219fec110. 02000000:00000001:3.0:1775767993.009507:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:1.0:1775767993.009514:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942057e2960. 00000020:00000001:1.0:1775767993.009517:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107591643720 : -112966117907896 : ffff994206aba248) 00000020:00001000:1.0:1775767993.009519:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff994206aba240 02000000:00000010:3.0:1775767993.009519:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b1000. 00000020:00000001:1.0:1775767993.009523:0:13471:0:(cl_object.c:151:cl_object_top()) ffff99420791a5f8 -> ffff99420791a5f8 00000020:00000001:1.0:1775767993.009526:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000100:00000001:3.0:1775767993.009527:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1775767993.009529:0:13471:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:1.0:1775767993.009532:0:13471:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x10176:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:1.0:1775767993.009535:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:1.0:1775767993.009538:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00100000:3.0:1775767993.009539:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8380 x1862022294260480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000080:00000001:1.0:1775767993.009540:0:13471:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:1.0:1775767993.009546:0:13471:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x10176:0x0]io ffff994205751e08 type 6 ignore/verify layout 1/0 00020000:00000002:1.0:1775767993.009551:0:13471:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207dfb6c8) owner 0000000000000000/ffff994220080000 00000100:00000001:3.0:1775767993.009552:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00020000:00000001:1.0:1775767993.009555:0:13471:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:1.0:1775767993.009558:0:13471:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00000100:00000001:3.0:1775767993.009558:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775767993.009562:0:13471:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00000100:00000040:3.0:1775767993.009563:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055f5621a] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8380 x1862022294260480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:1.0:1775767993.009564:0:13471:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775767993.009567:0:13471:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:1.0:1775767993.009571:0:13471:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:1.0:1775767993.009573:0:13471:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775767993.009577:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:1.0:1775767993.009579:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:1.0:1775767993.009580:0:13471:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00020000:00000002:1.0:1775767993.009583:0:13471:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207dfb6c8) owner 0000000000000000/ffff994220080000 00000020:00000001:1.0:1775767993.009586:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775767993.009587:0:13471:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:1.0:1775767993.009589:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00020000:00000001:1.0:1775767993.009591:0:13471:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00020000:00200000:1.0:1775767993.009593:0:13471:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00000001:3.0:1775767993.009596:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:1.0:1775767993.009598:0:13471:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00000100:00080000:3.0:1775767993.009599:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767993.009604:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775767993.009605:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767993.009606:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775767993.009608:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767993.009610:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5245 00020000:00000001:1.0:1775767993.009611:0:13471:0:(lov_io.c:155:lov_sub_get()) Process entered 00000400:00000200:3.0:1775767993.009612:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:1.0:1775767993.009614:0:13471:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000400:00000200:3.0:1775767993.009617:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000020:00000001:1.0:1775767993.009618:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00080000:3.0:1775767993.009622:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000020:00000010:1.0:1775767993.009623:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994206ababd0. 00000020:00000010:1.0:1775767993.009628:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994201343000. 00000100:00000040:3.0:1775767993.009628:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767993.009632:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775767993.009633:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000010:1.0:1775767993.009634:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994201343200. 00000020:00000010:1.0:1775767993.009641:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207cba120. 00000100:00000001:2.0:1775767993.009642:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000008:00000010:1.0:1775767993.009645:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207e00000. 00000100:00000040:3.0:1775767993.009646:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767993.009648:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00020000:00000010:1.0:1775767993.009651:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff9942031b30c0. 00000100:00000001:3.0:1775767993.009652:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00008000:00000010:1.0:1775767993.009656:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942054a8a98. 00000020:00000001:3.0:1775767993.009656:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767993.009659:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000080:00000010:1.0:1775767993.009660:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99422030b0a8. 00000020:00000001:3.0:1775767993.009661:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000080:00000010:1.0:1775767993.009666:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff994205753d98. 00000020:00000040:3.0:1775767993.009666:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000100:00000040:2.0:1775767993.009668:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a8380 x1862022294260480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1775767993.009671:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994201342e00. 00000020:00000001:3.0:1775767993.009671:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000008:00000010:1.0:1775767993.009677:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff994210709080. 00000020:00000001:3.0:1775767993.009678:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00020000:00000010:1.0:1775767993.009688:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff9942057418c0. 00000100:00000040:3.0:1775767993.009690:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00008000:00000010:1.0:1775767993.009695:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994219fec368. 00000100:00000001:3.0:1775767993.009697:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000080:00000010:1.0:1775767993.009702:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942057e24b0. 00000100:00000040:3.0:1775767993.009704:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000001:1.0:1775767993.009713:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107591646168 : -112966117905448 : ffff994206ababd8) 00000100:00000001:3.0:1775767993.009714:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00001000:1.0:1775767993.009723:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff994206ababd0 00000020:00000040:3.0:1775767993.009724:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000020:00000001:1.0:1775767993.009734:0:13471:0:(cl_object.c:151:cl_object_top()) ffff99420791ae90 -> ffff99420791a5f8 00000100:00080000:3.0:1775767993.009736:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000020:00000001:1.0:1775767993.009741:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000100:00000001:2.0:1775767993.009743:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767993.009743:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767993.009748:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767993.009749:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000020:00000001:1.0:1775767993.009751:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:1.0:1775767993.009753:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 02000000:00000001:2.0:1775767993.009754:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767993.009759:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1775767993.009760:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767993.009760:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:1.0:1775767993.009775:0:13471:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767993.009776:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767993.009777:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775767993.009789:0:13471:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107571233256 : -112966138318360 : ffff9942057431e8) 00000100:00000001:3.0:1775767993.009793:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000020:00000001:1.0:1775767993.009797:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000100:00100000:2.0:1775767993.009798:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294260480:192.168.202.156@tcp:250:kworker.0 00000020:00000001:1.0:1775767993.009801:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1775767993.009802:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00020000:00000001:1.0:1775767993.009814:0:13471:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767993.009816:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000008:00000001:3.0:1775767993.009816:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775767993.009818:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775767993.009823:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00000100:00000001:3.0:1775767993.009824:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775767993.009825:0:13471:0:(lov_io.c:1108:lov_io_lock()) Process entered 02000000:00000001:2.0:1775767993.009825:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00020000:00000001:1.0:1775767993.009834:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 02000000:00000001:2.0:1775767993.009835:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767993.009835:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8700. 00000020:00000001:1.0:1775767993.009837:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:1.0:1775767993.009841:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:1.0:1775767993.009848:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767993.009849:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:3.0:1775767993.009853:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:1.0:1775767993.009856:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767993.009858:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00020000:00000001:1.0:1775767993.009859:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767993.009861:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775767993.009862:0:13471:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775767993.009863:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1d000. 00000020:00000001:1.0:1775767993.009864:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000100:00000001:3.0:1775767993.009864:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767993.009868:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000020:00000001:1.0:1775767993.009869:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767993.009870:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775767993.009872:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767993.009873:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775767993.009880:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00020000:00000001:1.0:1775767993.009883:0:13471:0:(lov_io.c:1114:lov_io_start()) Process entered 02000000:00000010:3.0:1775767993.009883:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0c00. 00020000:00000001:1.0:1775767993.009886:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000001:3.0:1775767993.009888:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775767993.009889:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00000400:00000200:2.0:1775767993.009892:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00000001:1.0:1775767993.009893:0:13471:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000008:00000001:1.0:1775767993.009899:0:13471:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000100:00100000:3.0:1775767993.009899:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8700 x1862022294260608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:2.0:1775767993.009904:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000008:00000001:1.0:1775767993.009907:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:1.0:1775767993.009914:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000100:00000001:3.0:1775767993.009914:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000008:00000001:1.0:1775767993.009917:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767993.009918:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1775767993.009922:0:13471:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000400:00000010:2.0:1775767993.009922:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000040:3.0:1775767993.009924:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000faa25e3b] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8700 x1862022294260608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:1.0:1775767993.009927:0:13471:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000008:00000001:1.0:1775767993.009935:0:13471:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107754940544 : -112965954611072 : ffff994210675880) 00000008:00000020:1.0:1775767993.009940:0:13471:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994210675880 ready 0|-|- wr 1|-|+ rd 0|- 10 in flight 00000100:00000200:2.0:1775767993.009940:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294260480, portal 25 00000100:00000001:3.0:1775767993.009945:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767993.009949:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000008:00000001:1.0:1775767993.009950:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:1.0:1775767993.009954:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000100:00000001:2.0:1775767993.009955:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640853376 : -112966068698240 : ffff9942099a8380) 00000100:00080000:3.0:1775767993.009955:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000008:00000001:1.0:1775767993.009957:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:1.0:1775767993.009976:0:13471:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000008:00000020:1.0:1775767993.009980:0:13471:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994218232fc0 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000100:00080000:3.0:1775767993.009986:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000040:2.0:1775767993.009987:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a8380 x1862022294260480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768018 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:1.0:1775767993.009988:0:13471:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775767993.009991:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000008:00000001:1.0:1775767993.009996:0:13471:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767993.009997:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000020:00000001:1.0:1775767993.010000:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775767993.010008:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767993.010008:0:8:0:(import.c:484:import_select_connection()) Process entered 00020000:00000001:1.0:1775767993.010018:0:13471:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767993.010022:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5245 00000020:00000001:1.0:1775767993.010027:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775767993.010036:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00020000:00000001:1.0:1775767993.010040:0:13471:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00020000:00000001:1.0:1775767993.010047:0:13471:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000400:00000200:3.0:1775767993.010047:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767993.010048:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:1.0:1775767993.010052:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000001:1.0:1775767993.010058:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00000400:00000200:3.0:1775767993.010059:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000040:2.0:1775767993.010063:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00020000:00000001:1.0:1775767993.010064:0:13471:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775767993.010072:0:13471:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000400:00000010:2.0:1775767993.010072:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f880. 00000100:00080000:3.0:1775767993.010074:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000020:00000001:1.0:1775767993.010080:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00000100:00000200:2.0:1775767993.010085:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294260480, offset 0 00000020:00000001:1.0:1775767993.010086:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000100:00000040:3.0:1775767993.010086:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00020000:00000001:1.0:1775767993.010095:0:13471:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00000100:00000001:3.0:1775767993.010096:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00020000:00000001:1.0:1775767993.010099:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:1.0:1775767993.010105:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000100:00000040:3.0:1775767993.010105:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000200:2.0:1775767993.010109:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000020:00000001:1.0:1775767993.010111:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000100:00000001:3.0:1775767993.010115:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00020000:00000001:1.0:1775767993.010117:0:13471:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775767993.010123:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767993.010123:0:8:0:(genops.c:811:class_conn2export()) Process entered 00020000:00000001:1.0:1775767993.010127:0:13471:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000020:00000040:3.0:1775767993.010127:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000400:00000200:2.0:1775767993.010128:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:1.0:1775767993.010130:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:3.0:1775767993.010134:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000001:1.0:1775767993.010137:0:13471:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000020:00000040:3.0:1775767993.010139:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:1.0:1775767993.010141:0:13471:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000001:3.0:1775767993.010142:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:1.0:1775767993.010143:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000001:3.0:1775767993.010147:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00020000:00000001:1.0:1775767993.010148:0:13471:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00000400:00000200:2.0:1775767993.010148:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00020000:00000001:1.0:1775767993.010200:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:1.0:1775767993.010203:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000100:00000040:3.0:1775767993.010205:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000001:1.0:1775767993.010206:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000400:00000200:2.0:1775767993.010207:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00020000:00000001:1.0:1775767993.010210:0:13471:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767993.010210:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00020000:00000001:1.0:1775767993.010214:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767993.010217:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000100:00000040:3.0:1775767993.010218:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00020000:00000001:1.0:1775767993.010222:0:13471:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000020:00000001:1.0:1775767993.010229:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000100:00000001:3.0:1775767993.010229:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775767993.010234:0:13471:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767993.010236:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000020:00000040:3.0:1775767993.010237:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000020:00000001:1.0:1775767993.010246:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00000400:00000200:2.0:1775767993.010252:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00020000:00000001:1.0:1775767993.010253:0:13471:0:(lov_io.c:675:lov_io_fini()) Process entered 00000100:00080000:3.0:1775767993.010253:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00020000:00000001:1.0:1775767993.010259:0:13471:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:1.0:1775767993.010263:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00000100:00000001:3.0:1775767993.010264:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775767993.010266:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:1.0:1775767993.010271:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff994206ababd0 00000400:00000200:2.0:1775767993.010272:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225208:225208:256:4294967295] 192.168.202.156@tcp LPNI seq info [225204:225204:8:4294967295] 02000000:00000001:3.0:1775767993.010272:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000020:00000010:1.0:1775767993.010281:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994201343200. 02000000:00000001:3.0:1775767993.010281:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1775767993.010290:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:3.0:1775767993.010290:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1775767993.010296:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207cba120. 00000100:00000001:3.0:1775767993.010303:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00000010:1.0:1775767993.010304:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207e00000. 00000008:00100000:3.0:1775767993.010307:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00020000:00000010:1.0:1775767993.010310:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff9942031b30c0. 00000008:00000001:3.0:1775767993.010312:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:1.0:1775767993.010319:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942054a8a98. 00000100:00000001:3.0:1775767993.010321:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:1.0:1775767993.010331:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99422030b0a8. 00000400:00000200:2.0:1775767993.010331:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000100:00000010:3.0:1775767993.010332:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a9180. 00000020:00000040:3.0:1775767993.010345:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000080:00000010:1.0:1775767993.010346:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff994205753d98. 00000800:00000200:2.0:1775767993.010355:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 02000000:00000001:3.0:1775767993.010355:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000020:00000010:1.0:1775767993.010356:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994201343000. 02000000:00000001:3.0:1775767993.010360:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767993.010368:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000008:00000010:1.0:1775767993.010369:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff994210709080. 00000800:00000010:2.0:1775767993.010370:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3a00. 02000000:00000001:3.0:1775767993.010370:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775767993.010377:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:1.0:1775767993.010379:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff9942057418c0. 00008000:00000010:1.0:1775767993.010382:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994219fec368. 02000000:00000010:3.0:1775767993.010382:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2000. 00000800:00000200:2.0:1775767993.010383:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000080:00000010:1.0:1775767993.010391:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942057e24b0. 00000100:00000001:3.0:1775767993.010392:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1775767993.010394:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000020:00000010:1.0:1775767993.010396:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994201342e00. 00000020:00000010:1.0:1775767993.010400:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994206ababd0. 00000100:00100000:3.0:1775767993.010401:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a9180 x1862022294260736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00020000:00000001:1.0:1775767993.010407:0:13471:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:1.0:1775767993.010411:0:13471:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000100:00000001:3.0:1775767993.010417:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000080:00000001:1.0:1775767993.010421:0:13471:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000100:00000001:3.0:1775767993.010421:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775767993.010421:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000080:00200000:1.0:1775767993.010426:0:13471:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x10176:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00000040:3.0:1775767993.010433:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bdc2586] to pc [ptlrpcd_rcv+-1] req@ffff9942099a9180 x1862022294260736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000001:1.0:1775767993.010435:0:13471:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000800:00000200:2.1:1775767993.010435:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000020:00000001:1.0:1775767993.010437:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:1.0:1775767993.010442:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff994206aba240 00000020:00000010:1.0:1775767993.010447:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994208a8ec00. 00000020:00000010:1.0:1775767993.010456:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207cba9c0. 00000800:00000100:2.0:1775767993.010462:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000008:00000010:1.0:1775767993.010466:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207e02ca0. 00000800:00000010:2.0:1775767993.010471:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3a00. 00020000:00000010:1.0:1775767993.010472:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff9942031b3480. 00000100:00000001:3.0:1775767993.010475:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:1.0:1775767993.010482:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942054ab170. 00000100:00080000:3.0:1775767993.010482:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:2.0:1775767993.010484:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000080:00000010:1.0:1775767993.010488:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99422030bf18. 00000080:00000010:1.0:1775767993.010498:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff994205751da8. 00000100:00080000:3.0:1775767993.010498:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775767993.010502:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000020:00000010:1.0:1775767993.010505:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994208a8f600. 00000400:00000200:2.0:1775767993.010507:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000100:00080000:3.0:1775767993.010507:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000008:00000010:1.0:1775767993.010510:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff994210709ce0. 00020000:00000010:1.0:1775767993.010520:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994205743180. 00000100:00000001:3.0:1775767993.010522:0:8:0:(import.c:484:import_select_connection()) Process entered 00008000:00000010:1.0:1775767993.010524:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994219fec110. 00000400:00000200:2.0:1775767993.010524:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f880 00000100:00080000:3.0:1775767993.010529:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5245 00000080:00000010:1.0:1775767993.010537:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942057e2960. 00000400:00000010:2.0:1775767993.010537:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f880. 00000400:00000200:3.0:1775767993.010545:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000010:1.0:1775767993.010547:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994208a8e000. 00000100:00000001:2.0:1775767993.010552:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000020:00000010:1.0:1775767993.010553:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994206aba240. 00000400:00000200:3.0:1775767993.010558:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000080:00000001:1.0:1775767993.010561:0:13471:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775767993.010561:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000200:2.0:1775767993.010563:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a8380 x1862022294260480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768018 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000001:1.0:1775767993.010569:0:13471:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767993.010573:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767993.010583:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767993.010596:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767993.010599:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1775767993.010602:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000040:2.0:1775767993.010606:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a8380 x1862022294260480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768018 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775767993.010608:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767993.010614:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:3.0:1775767993.010620:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:2.0:1775767993.010623:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775767993.010624:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000100:00000001:2.0:1775767993.010626:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:3.0:1775767993.010633:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:2.0:1775767993.010635:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775767993.010641:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000400:00000200:2.0:1775767993.010649:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000040:3.0:1775767993.010654:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775767993.010656:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299919166 00000100:00000001:3.0:1775767993.010659:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767993.010663:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000010:2.0:1775767993.010665:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d6840. 00000100:00000001:3.0:1775767993.010666:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775767993.010673:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:3.0:1775767993.010677:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775767993.010679:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1775767993.010680:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767993.010685:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1775767993.010689:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299919166 02000000:00000001:3.0:1775767993.010690:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767993.010697:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775767993.010699:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d61e0. 00000100:00000001:3.0:1775767993.010700:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767993.010706:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000100:00000001:2.0:1775767993.010707:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000008:00000001:3.0:1775767993.010712:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767993.010713:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767993.010723:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767993.010723:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767993.010734:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000010:3.0:1775767993.010736:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ab480. 00000100:00100000:2.0:1775767993.010741:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767993/real 1775767993] req@ffff9942099a8380 x1862022294260480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1775767993.010744:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:3.0:1775767993.010753:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775767993.010759:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1775767993.010762:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767993.010766:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1775767993.010766:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767993.010773:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:2.0:1775767993.010774:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775767993.010782:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 02000000:00000001:3.0:1775767993.010783:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775767993.010785:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775767993.010790:0:2399:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000010:3.0:1775767993.010793:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b3400. 00000100:00000200:2.0:1775767993.010794:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a8380 x1862022294260480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767993.010802:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1775767993.010808:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a8380 x1862022294260480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767993.010815:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ab480 x1862022294260864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775767993.010821:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767993.010824:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767993.010830:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767993.010836:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767993.010848:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a8380 x1862022294260480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767993.010848:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775767993.010855:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767993.010871:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000067a0ba74] to pc [ptlrpcd_rcv+-1] req@ffff9942099ab480 x1862022294260864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767993.010875:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767993.010888:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a8380 x1862022294260480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768018 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767993.010890:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767993.010896:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767993.010910:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775767993.010915:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775767993.010918:0:8:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775767993.010928:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775767993.010929:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775767993.010935:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775767993.010937:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5245 00000100:00000001:2.0:1775767993.010939:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767993.010944:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767993.010948:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000400:00000200:3.0:1775767993.010951:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767993.010952:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767993.010977:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000400:00000200:3.0:1775767993.010983:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775767993.010988:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00080000:3.0:1775767993.010990:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1775767993.010991:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767993.011007:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a8380 x1862022294260480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768018 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775767993.011007:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767993.011019:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767993.011029:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00100000:2.0:1775767993.011033:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294260480:192.168.202.156@tcp:250:kworker.0 00000100:00000001:3.0:1775767993.011040:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:2.0:1775767993.011042:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1775767993.011049:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:2.0:1775767993.011050:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000020:00000040:3.0:1775767993.011054:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000100:00000040:2.0:1775767993.011058:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a8380 x1862022294260480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768018 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775767993.011059:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767993.011064:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:3.0:1775767993.011075:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000001:2.0:1775767993.011077:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767993.011081:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000001:3.0:1775767993.011082:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 02000000:00000010:2.0:1775767993.011085:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1d000. 02000000:00000001:2.0:1775767993.011091:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767993.011095:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000040:3.0:1775767993.011095:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000040:2.0:1775767993.011098:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000100:00000001:3.0:1775767993.011103:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:2.0:1775767993.011104:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767993.011108:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b1000. 00000100:00000040:3.0:1775767993.011113:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000001:2.0:1775767993.011116:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767993.011123:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1775767993.011123:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000010:2.0:1775767993.011127:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a8380. 00000100:00000001:2.0:1775767993.011133:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000020:00000040:3.0:1775767993.011133:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00000001:2.0:1775767993.011137:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767993.011144:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775767993.011144:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775767993.011152:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767993.011153:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767993.011177:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767993.011180:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1775767993.011181:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775767993.011187:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767993.011188:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1775767993.011189:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1775767993.011195:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a8700 x1862022294260608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1775767993.011197:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767993.011206:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775767993.011212:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000008:00100000:3.0:1775767993.011213:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000100:00000001:2.0:1775767993.011215:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767993.011223:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000001:3.0:1775767993.011224:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767993.011226:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767993.011235:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767993.011235:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767993.011247:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294260608:192.168.202.156@tcp:38:kworker.0 00000100:00000010:3.0:1775767993.011249:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8e00. 00000100:00000001:2.0:1775767993.011251:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767993.011255:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:3.0:1775767993.011256:0:8:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:2.0:1775767993.011257:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767993.011264:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1775767993.011267:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000010:2.0:1775767993.011268:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1d000. 02000000:00000001:3.0:1775767993.011270:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767993.011272:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767993.011276:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767993.011280:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:2.0:1775767993.011281:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:3.0:1775767993.011286:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775767993.011287:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775767993.011297:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 02000000:00000010:3.0:1775767993.011298:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0400. 00000100:00000200:2.0:1775767993.011301:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294260608, portal 10 00000100:00000001:2.0:1775767993.011305:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767993.011306:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640854272 : -112966068697344 : ffff9942099a8700) 00000100:00000001:3.0:1775767993.011306:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767993.011324:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a8700 x1862022294260608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768018 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767993.011325:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8e00 x1862022294260992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775767993.011369:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767993.011382:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000100:00000001:3.0:1775767993.011382:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000010:2.0:1775767993.011386:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f880. 00000100:00000001:3.0:1775767993.011387:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1775767993.011396:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294260608, offset 0 00000100:00000040:3.0:1775767993.011400:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000864a6866] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8e00 x1862022294260992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1775767993.011411:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767993.011432:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775767993.011435:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767993.011440:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767993.011445:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:3.0:1775767993.011446:0:8:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5255) 00000400:00000200:2.0:1775767993.011450:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767993.011453:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767993.011456:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767993.011458:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225209:225209:256:4294967295] 192.168.202.156@tcp LPNI seq info [225205:225205:8:4294967295] 00000400:00000200:2.0:1775767993.011465:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767993.011471:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767993.011475:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767993.011479:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3a00. 00000800:00000200:2.0:1775767993.011482:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775767993.011484:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775767993.011488:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767993.011490:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000100:2.0:1775767993.011493:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767993.011494:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3a00. 00000400:00000200:2.0:1775767993.011496:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775767993.011499:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775767993.011501:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f880 00000400:00000010:2.0:1775767993.011502:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f880. 00000100:00000001:2.0:1775767993.011504:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767993.011507:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a8700 x1862022294260608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768018 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767993.011514:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767993.011516:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a8700 x1862022294260608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768018 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767993.011520:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767993.011522:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775767993.011524:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767993.011527:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:2.0:1775767993.011529:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767993.011531:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299917844 00000100:00000001:2.0:1775767993.011533:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299919166 00000100:00000001:2.0:1775767993.011535:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299917844 00000100:00000001:2.0:1775767993.011537:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299919166 00000100:00000001:2.0:1775767993.011538:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767993.011540:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767993.011541:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767993.011543:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767993/real 1775767993] req@ffff9942099a8700 x1862022294260608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775767993.011548:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767993.011549:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767993.011550:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775767993.011552:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775767993.011553:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775767993.011554:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767993.011556:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a8700 x1862022294260608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767993.011561:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a8700 x1862022294260608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767993.011565:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767993.011566:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767993.011567:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767993.011568:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767993.011570:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a8700 x1862022294260608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767993.011574:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767993.011576:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a8700 x1862022294260608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768018 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767993.011582:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767993.011583:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767993.011585:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767993.011586:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767993.011587:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767993.011590:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767993.011592:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767993.011594:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767993.011595:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767993.011598:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a8700 x1862022294260608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768018 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767993.011602:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294260608:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775767993.011606:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767993.011608:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a9180 x1862022294260736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767993.011613:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767993.011614:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767993.011615:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767993.011617:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767993.011618:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767993.011620:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294260736:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775767993.011622:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767993.011623:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767993.011625:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767993.011626:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775767993.011628:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f800. 02000000:00000001:2.0:1775767993.011630:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767993.011632:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767993.011634:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775767993.011636:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775767993.011638:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294260736, portal 10 00000100:00000001:2.0:1775767993.011639:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767993.011641:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640856960 : -112966068694656 : ffff9942099a9180) 00000100:00000040:2.0:1775767993.011645:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a9180 x1862022294260736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768018 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767993.011653:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767993.011655:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767993.011658:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f880. 00000100:00000200:2.0:1775767993.011660:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294260736, offset 0 00000400:00000200:2.0:1775767993.011664:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767993.011669:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767993.011672:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767993.011676:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767993.011678:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767993.011681:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767993.011683:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767993.011687:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225210:225210:256:4294967295] 192.168.202.156@tcp LPNI seq info [225206:225206:8:4294967295] 00000400:00000200:2.0:1775767993.011692:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767993.011699:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767993.011704:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767993.011707:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3a00. 00000800:00000200:2.0:1775767993.011710:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775767993.011712:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775767993.011715:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767993.011717:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000100:2.0:1775767993.011719:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767993.011721:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3a00. 00000400:00000200:2.0:1775767993.011723:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775767993.011726:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775767993.011728:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f880 00000400:00000010:2.0:1775767993.011729:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f880. 00000100:00000001:2.0:1775767993.011731:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767993.011733:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a9180 x1862022294260736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768018 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767993.011744:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767993.011746:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a9180 x1862022294260736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768018 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767993.011750:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767993.011751:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775767993.011753:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767993.011755:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:2.0:1775767993.011757:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767993.011758:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299917836 00000100:00000001:2.0:1775767993.011760:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299919166 00000100:00000001:2.0:1775767993.011762:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299917836 00000100:00000001:2.0:1775767993.011764:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299919166 00000100:00000001:2.0:1775767993.011765:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767993.011766:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767993.011768:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767993.011770:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767993/real 1775767993] req@ffff9942099a9180 x1862022294260736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775767993.011774:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767993.011775:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767993.011776:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775767993.011778:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775767993.011779:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775767993.011780:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767993.011782:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a9180 x1862022294260736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767993.011788:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a9180 x1862022294260736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767993.011791:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767993.011793:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767993.011794:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767993.011795:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767993.011797:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a9180 x1862022294260736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767993.011802:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767993.011804:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a9180 x1862022294260736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768018 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767993.011808:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767993.011810:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767993.011811:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767993.011811:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767993.011812:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767993.011814:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767993.011816:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767993.011818:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767993.011819:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767993.011821:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a9180 x1862022294260736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768018 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767993.011826:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294260736:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775767993.011828:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767993.011831:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ab480 x1862022294260864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767993.011836:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767993.011837:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767993.011838:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767993.011839:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767993.011841:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767993.011843:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ab480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294260864:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767993.011845:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767993.011847:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767993.011848:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767993.011849:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775767993.011851:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1e400. 02000000:00000001:2.0:1775767993.011852:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767993.011854:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767993.011856:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775767993.011858:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775767993.011860:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294260864, portal 4 00000100:00000001:2.0:1775767993.011862:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767993.011863:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640865920 : -112966068685696 : ffff9942099ab480) 00000100:00000040:2.0:1775767993.011869:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ab480 x1862022294260864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768018 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767993.011875:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767993.011878:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767993.011880:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f880. 00000100:00000200:2.0:1775767993.011883:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294260864, offset 0 00000400:00000200:2.0:1775767993.011887:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767993.011891:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767993.011895:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767993.011899:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767993.011901:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767993.011904:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767993.011907:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767993.011910:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225211:225211:256:4294967295] 192.168.202.156@tcp LPNI seq info [225207:225207:8:4294967295] 00000400:00000200:2.0:1775767993.011915:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767993.011921:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767993.011927:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767993.011930:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3a00. 00000800:00000200:2.0:1775767993.011935:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775767993.011938:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775767993.011942:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767993.011944:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000100:2.0:1775767993.011948:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767993.011951:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3a00. 00000400:00000200:2.0:1775767993.011953:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775767993.011957:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775767993.011974:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f880 00000400:00000010:2.0:1775767993.011976:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f880. 00000100:00000001:2.0:1775767993.011979:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767993.011983:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ab480 x1862022294260864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768018 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767993.011990:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767993.011993:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ab480 x1862022294260864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768018 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767993.011997:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767993.011999:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775767993.012001:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767993.012004:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:2.0:1775767993.012006:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767993.012008:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299917859 00000100:00000001:2.0:1775767993.012010:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299919166 00000100:00000001:2.0:1775767993.012011:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299917859 00000100:00000001:2.0:1775767993.012013:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299919166 00000100:00000001:2.0:1775767993.012015:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767993.012016:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767993.012018:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767993.012020:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767993/real 1775767993] req@ffff9942099ab480 x1862022294260864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775767993.012025:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767993.012026:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767993.012027:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775767993.012029:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775767993.012030:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775767993.012031:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767993.012034:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ab480 x1862022294260864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767993.012039:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ab480 x1862022294260864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767993.012042:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767993.012043:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767993.012044:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767993.012045:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767993.012047:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ab480 x1862022294260864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767993.012051:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767993.012054:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ab480 x1862022294260864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768018 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767993.012059:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767993.012060:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767993.012061:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767993.012062:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767993.012063:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767993.012066:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767993.012068:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767993.012070:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767993.012071:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767993.012073:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ab480 x1862022294260864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768018 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767993.012082:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ab480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294260864:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767993.012085:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767993.012087:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767993.012088:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a8700 x1862022294260608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768018 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767993.012092:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767993.012093:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767993.012095:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1d000. 02000000:00000001:2.0:1775767993.012096:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767993.012097:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767993.012098:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:2.0:1775767993.012100:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767993.012101:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b0c00. 02000000:00000001:2.0:1775767993.012103:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767993.012104:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767993.012105:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a8700. 00000100:00000001:2.0:1775767993.012107:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767993.012108:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767993.012109:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767993.012111:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a9180 x1862022294260736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768018 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767993.012116:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767993.012117:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767993.012118:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f800. 02000000:00000001:2.0:1775767993.012119:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767993.012120:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767993.012121:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:2.0:1775767993.012122:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767993.012123:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b2000. 02000000:00000001:2.0:1775767993.012124:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767993.012125:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767993.012125:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a9180. 00000100:00000001:2.0:1775767993.012128:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767993.012128:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767993.012130:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767993.012131:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ab480 x1862022294260864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768018 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767993.012136:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767993.012137:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767993.012137:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1e400. 02000000:00000001:2.0:1775767993.012139:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767993.012140:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767993.012141:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:2.0:1775767993.012142:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767993.012142:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b3400. 02000000:00000001:2.0:1775767993.012144:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767993.012144:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767993.012145:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ab480. 00000100:00000001:2.0:1775767993.012147:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767993.012148:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767993.012149:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767993.012151:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767993.012152:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767993.012153:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775767993.012170:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767993.012171:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775767993.012174:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a8e00 x1862022294260992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767993.012178:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767993.012179:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767993.012181:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775767993.012182:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775767993.012183:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767993.012185:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294260992:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767993.012187:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775767993.012189:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775767993.012190:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775767993.012191:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775767993.012194:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1e400. 02000000:00000001:2.0:1775767993.012195:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775767993.012198:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775767993.012199:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775767993.012201:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775767993.012203:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294260992, portal 4 00000100:00000001:2.0:1775767993.012205:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775767993.012206:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640856064 : -112966068695552 : ffff9942099a8e00) 00000100:00000040:2.0:1775767993.012210:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a8e00 x1862022294260992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768018 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767993.012214:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775767993.012216:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775767993.012218:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f880. 00000100:00000200:2.0:1775767993.012219:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294260992, offset 0 00000400:00000200:2.0:1775767993.012222:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775767993.012224:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767993.012226:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775767993.012228:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775767993.012230:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775767993.012231:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775767993.012233:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775767993.012235:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225212:225212:256:4294967295] 192.168.202.156@tcp LPNI seq info [225208:225208:8:4294967295] 00000400:00000200:2.0:1775767993.012238:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775767993.012242:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775767993.012245:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767993.012247:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3a00. 00000800:00000200:2.0:1775767993.012249:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775767993.012251:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775767993.012254:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775767993.012256:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000100:2.0:1775767993.012259:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775767993.012260:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3a00. 00000400:00000200:2.0:1775767993.012262:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775767993.012265:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775767993.012267:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f880 00000400:00000010:2.0:1775767993.012268:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f880. 00000100:00000001:2.0:1775767993.012269:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775767993.012271:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a8e00 x1862022294260992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768018 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767993.012277:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767993.012279:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a8e00 x1862022294260992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768018 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767993.012283:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767993.012284:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775767993.012286:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775767993.012288:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:2.0:1775767993.012290:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775767993.012291:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299917863 00000100:00000001:2.0:1775767993.012293:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299919166 00000100:00000001:2.0:1775767993.012295:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299917863 00000100:00000001:2.0:1775767993.012297:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299919166 00000100:00000001:2.0:1775767993.012298:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775767993.012299:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767993.012300:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775767993.012302:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767993/real 1775767993] req@ffff9942099a8e00 x1862022294260992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775767993.012307:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775767993.012308:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775767993.012308:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775767993.012310:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775767993.012310:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775767993.012312:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775767993.012314:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a8e00 x1862022294260992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775767993.012320:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a8e00 x1862022294260992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767993.012324:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775767993.012325:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767993.012326:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767993.012327:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775767993.012329:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a8e00 x1862022294260992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767993.012333:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775767993.012335:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a8e00 x1862022294260992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768018 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767993.012339:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775767993.012340:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767993.012341:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775767993.012342:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767993.012343:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775767993.012345:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775767993.012346:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767993.012348:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775767993.012350:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775767993.012352:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a8e00 x1862022294260992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768018 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767993.012357:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294260992:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775767993.012359:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767993.012361:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775767993.012362:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a8e00 x1862022294260992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768018 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775767993.012367:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775767993.012368:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775767993.012369:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1e400. 02000000:00000001:2.0:1775767993.012371:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775767993.012372:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775767993.012372:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:2.0:1775767993.012373:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775767993.012374:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b0400. 02000000:00000001:2.0:1775767993.012376:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775767993.012377:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775767993.012378:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a8e00. 00000100:00000001:2.0:1775767993.012379:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775767993.012381:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767993.012382:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767993.012385:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767993.012387:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767993.012389:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775767993.012390:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767993.073361:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767993.073366:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767993.073372:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767993.073375:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767993.073379:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767993.073380:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767993.073384:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767993.073386:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767993.073431:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767993.073432:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767993.073475:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767993.073491:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767993.073494:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767993.137256:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767993.137262:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767993.137264:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767993.137268:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767993.137271:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767993.137274:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767993.137276:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767993.137279:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767993.137281:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767994.033257:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767994.033257:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767994.033264:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767994.033265:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767994.033267:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767994.033268:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767994.033276:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:0.0:1775767994.033277:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775767994.033282:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767994.033283:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767994.033284:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767994.033288:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767994.033289:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767994.033293:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767994.033293:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767994.033297:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767994.033298:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767994.033301:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767994.033301:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767994.033306:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767994.033309:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767994.033309:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775767994.033316:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767994.033321:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767994.033324:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767994.033328:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767994.033332:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767994.097286:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767994.097292:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767994.097298:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767994.097299:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767994.097302:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767994.097304:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767994.097306:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767994.097307:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767994.097310:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767994.097311:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767994.097331:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767994.097333:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767994.097336:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767994.161299:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767994.161307:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767994.161309:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767994.161315:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767994.161319:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767994.161323:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767994.161326:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767994.161328:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767994.161331:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767994.353283:0:14953:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4299919166, at 4299919168 00000100:00000040:1.0:1775767994.353288:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299919166, at 4299919168 00000100:00000010:0.0:1775767994.353300:0:14953:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d6840. 00000100:00000010:1.0:1775767994.353303:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d61e0. 00000100:00000001:0.0:1775767995.057271:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767995.057271:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767995.057280:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767995.057280:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767995.057284:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767995.057285:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775767995.057294:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775767995.057294:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:0.0:1775767995.057295:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767995.057305:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767995.057307:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767995.057307:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767995.057310:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767995.057313:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767995.057314:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767995.057314:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767995.057319:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767995.057323:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767995.057325:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:0.0:1775767995.057326:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767995.057330:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767995.057333:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767995.057335:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767995.057340:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767995.057346:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767995.057350:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767995.057353:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767995.121294:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767995.121303:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767995.121312:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767995.121314:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767995.121320:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767995.121322:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767995.121327:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767995.121329:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767995.121333:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767995.121334:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767995.121356:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767995.121358:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767995.121360:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767995.185313:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767995.185321:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767995.185323:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767995.185330:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767995.185334:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767995.185340:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767995.185343:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767995.185346:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767995.185350:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767996.081305:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775767996.081305:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767996.081306:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767996.081319:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767996.081319:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767996.081319:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767996.081322:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767996.081328:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767996.081328:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767996.081329:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775767996.081340:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1775767996.081344:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775767996.081344:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:0.0:1775767996.081347:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767996.081357:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767996.081360:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767996.081360:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767996.081362:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767996.081369:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767996.081370:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767996.081381:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767996.081382:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767996.081386:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767996.081390:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767996.081391:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767996.081397:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767996.081402:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767996.145265:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767996.145270:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767996.145274:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767996.145277:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775767996.145279:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775767996.145282:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767996.145284:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775767996.145286:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767996.145288:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767996.145289:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767996.145292:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767996.145293:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775767996.145294:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775767996.145295:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767996.145296:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775767996.145297:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767996.145298:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767996.145300:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767996.145302:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767996.145303:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775767996.145304:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775767996.145305:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767996.145306:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775767996.145307:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767996.145308:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767996.145310:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767996.145312:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767996.145313:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775767996.145314:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775767996.145315:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767996.145316:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775767996.145317:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767996.145318:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767996.145319:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767996.145333:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767996.145335:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767996.145337:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767996.209285:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767996.209290:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767996.209292:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767996.209297:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767996.209300:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767996.209304:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767996.209306:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767996.209308:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767996.209309:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767997.105275:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767997.105286:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767997.105288:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767997.105294:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775767997.105299:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767997.105299:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767997.105304:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767997.105305:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767997.105310:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767997.105311:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767997.105315:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:1.0:1775767997.105318:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775767997.105322:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767997.105322:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775767997.105323:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767997.105329:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767997.105329:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767997.105338:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767997.105342:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767997.105343:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767997.105347:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775767997.105353:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775767997.105358:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767997.105376:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767997.105379:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775767997.105381:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767997.105384:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767997.169267:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767997.169272:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767997.169276:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767997.169278:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767997.169283:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767997.169284:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767997.169286:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767997.169287:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767997.169290:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767997.169291:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767997.169312:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767997.169315:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767997.169318:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767997.233258:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767997.233264:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767997.233265:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767997.233269:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767997.233273:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767997.233276:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767997.233278:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767997.233280:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767997.233282:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767998.129241:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767998.129241:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767998.129247:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767998.129248:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767998.129250:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767998.129250:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775767998.129252:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767998.129255:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:2.0:1775767998.129256:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:0.0:1775767998.129258:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775767998.129259:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767998.129263:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767998.129264:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767998.129264:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775767998.129265:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767998.129268:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767998.129270:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775767998.129271:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767998.129271:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767998.129271:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1775767998.129276:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767998.129277:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1775767998.129277:0:8:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775767998.129280:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767998.129281:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767998.129284:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775767998.129284:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:0.0:1775767998.129289:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767998.129293:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767998.129293:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:3.0:1775767998.129293:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5250 00000100:00000001:1.0:1775767998.129297:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767998.129301:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775767998.129306:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775767998.129312:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767998.129316:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775767998.129319:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000080:00000001:2.0:1775767998.129320:0:13471:0:(rw.c:1481:ll_writepages()) Process entered 00000100:00000001:3.0:1775767998.129323:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000080:00000001:2.0:1775767998.129324:0:13471:0:(file.c:5250:cl_sync_file_range()) Process entered 00000020:00000001:2.0:1775767998.129327:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00000040:3.0:1775767998.129327:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767998.129331:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775767998.129335:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767998.129336:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000010:2.0:1775767998.129337:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994210b32630. 00000020:00000001:3.0:1775767998.129338:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775767998.129342:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000010:2.0:1775767998.129344:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994209d2d600. 00000020:00000001:3.0:1775767998.129344:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:3.0:1775767998.129346:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:3.0:1775767998.129350:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767998.129351:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767998.129353:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775767998.129354:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000010:2.0:1775767998.129356:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994209d2de00. 00000020:00000040:3.0:1775767998.129356:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:3.0:1775767998.129361:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000020:00000010:2.0:1775767998.129362:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942182d6300. 00000100:00000001:3.0:1775767998.129363:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767998.129367:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000008:00000010:2.0:1775767998.129369:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207ccc2f0. 02000000:00000001:3.0:1775767998.129369:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775767998.129374:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:2.0:1775767998.129375:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994220156e40. 00000100:00000001:3.0:1775767998.129377:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775767998.129380:0:8:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:2.0:1775767998.129382:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942077a8e20. 00000100:00000010:3.0:1775767998.129385:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8a80. 00000080:00000010:2.0:1775767998.129388:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff9942087472a0. 00000020:00000040:3.0:1775767998.129389:0:8:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:3.0:1775767998.129394:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000080:00000010:2.0:1775767998.129395:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b8000. 02000000:00000001:3.0:1775767998.129397:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1775767998.129400:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994209d2d000. 00000100:00000001:3.0:1775767998.129400:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767998.129404:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000008:00000010:2.0:1775767998.129405:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942077cd600. 02000000:00000001:3.0:1775767998.129408:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:2.0:1775767998.129409:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff99420876d680. 00008000:00000010:2.0:1775767998.129413:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076cf60. 02000000:00000010:3.0:1775767998.129416:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2800. 00000080:00000010:2.0:1775767998.129417:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff994205066870. 00000020:00000001:2.0:1775767998.129421:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107759908408 : -112965949643208 : ffff994210b32638) 00000020:00001000:2.0:1775767998.129423:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff994210b32630 00000100:00000001:3.0:1775767998.129424:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767998.129427:0:13471:0:(cl_object.c:151:cl_object_top()) ffff9942078a76c0 -> ffff9942078a76c0 00000020:00000001:2.0:1775767998.129430:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:2.0:1775767998.129432:0:13471:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000100:00100000:3.0:1775767998.129433:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8a80 x1862022294261120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000080:00200000:2.0:1775767998.129435:0:13471:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:2.0:1775767998.129442:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775767998.129444:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:2.0:1775767998.129446:0:13471:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767998.129447:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00020000:00000002:2.0:1775767998.129450:0:13471:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017a:0x0]io ffff9942077b8060 type 6 ignore/verify layout 1/0 00000100:00000001:3.0:1775767998.129452:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775767998.129456:0:13471:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207c1f9b0) owner 0000000000000000/ffff994220080000 00000100:00000040:3.0:1775767998.129459:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006c3265af] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8a80 x1862022294261120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:2.0:1775767998.129461:0:13471:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:2.0:1775767998.129463:0:13471:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:2.0:1775767998.129467:0:13471:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00020000:00000001:2.0:1775767998.129474:0:13471:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767998.129478:0:13471:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:2.0:1775767998.129481:0:13471:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:2.0:1775767998.129483:0:13471:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767998.129483:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767998.129485:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000100:00080000:3.0:1775767998.129487:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000001:2.0:1775767998.129488:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:2.0:1775767998.129491:0:13471:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00000100:00080000:3.0:1775767998.129492:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775767998.129495:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00020000:00000002:2.0:1775767998.129496:0:13471:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207c1f9b0) owner 0000000000000000/ffff994220080000 00000100:00080000:3.0:1775767998.129497:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000020:00000001:2.0:1775767998.129503:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767998.129503:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775767998.129506:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5250 00000020:00000001:2.0:1775767998.129508:0:13471:0:(cl_io.c:823:cl_io_loop()) Process entered 00000400:00000200:3.0:1775767998.129511:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:2.0:1775767998.129512:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00020000:00000001:2.0:1775767998.129516:0:13471:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00020000:00200000:2.0:1775767998.129520:0:13471:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00000001:0.0:1775767998.129522:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:3.0:1775767998.129523:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:0.0:1775767998.129526:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00020000:00000002:2.0:1775767998.129526:0:13471:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00000100:00000001:0.0:1775767998.129530:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00080000:3.0:1775767998.129531:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00020000:00000001:2.0:1775767998.129534:0:13471:0:(lov_io.c:155:lov_sub_get()) Process entered 00000100:00000040:3.0:1775767998.129535:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00020000:00000001:2.0:1775767998.129538:0:13471:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000020:00000001:2.0:1775767998.129541:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00000001:3.0:1775767998.129541:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775767998.129542:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a8a80 x1862022294261120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775767998.129546:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000010:2.0:1775767998.129547:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994210b32090. 00000100:00000001:3.0:1775767998.129552:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000010:2.0:1775767998.129554:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994209d2dc00. 00000020:00000001:3.0:1775767998.129559:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000010:2.0:1775767998.129560:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994209d2cc00. 00000020:00000040:3.0:1775767998.129562:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000010:2.0:1775767998.129566:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942182d6480. 00000020:00000001:3.0:1775767998.129566:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:0.0:1775767998.129567:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000040:3.0:1775767998.129569:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000100:00000001:0.0:1775767998.129571:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:2.0:1775767998.129571:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207cc8b28. 00000020:00000001:3.0:1775767998.129572:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00020000:00000010:2.0:1775767998.129577:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994220156840. 02000000:00000001:0.0:1775767998.129578:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:3.0:1775767998.129578:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 02000000:00000001:0.0:1775767998.129583:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00008000:00000010:2.0:1775767998.129584:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942077a9530. 00000100:00000040:3.0:1775767998.129585:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 02000000:00000001:0.0:1775767998.129590:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:2.0:1775767998.129591:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994208747738. 00000100:00000001:3.0:1775767998.129591:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775767998.129596:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000080:00000010:2.0:1775767998.129597:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077ba910. 00000100:00000001:3.0:1775767998.129599:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00100000:0.0:1775767998.129601:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294261120:192.168.202.156@tcp:250:kworker.0 00000020:00000010:2.0:1775767998.129602:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9942060d5800. 00000020:00000040:3.0:1775767998.129602:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000008:00000010:2.0:1775767998.129609:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942077cd8c0. 00000100:00080000:3.0:1775767998.129609:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775767998.129612:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00020000:00000010:2.0:1775767998.129613:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff99420876e400. 00000100:00000001:3.0:1775767998.129614:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767998.129618:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775767998.129618:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00008000:00000010:2.0:1775767998.129619:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076c3f0. 02000000:00000001:0.0:1775767998.129623:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:2.0:1775767998.129624:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942050663c0. 02000000:00000001:3.0:1775767998.129626:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775767998.129630:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:2.0:1775767998.129630:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107759906968 : -112965949644648 : ffff994210b32098) 02000000:00000001:3.0:1775767998.129631:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:2.0:1775767998.129635:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff994210b32090 00000100:00000001:3.0:1775767998.129635:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000010:0.0:1775767998.129639:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002bc00. 00000020:00000001:2.0:1775767998.129639:0:13471:0:(cl_object.c:151:cl_object_top()) ffff9942078a7788 -> ffff9942078a76c0 00000008:00100000:3.0:1775767998.129639:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000020:00000001:2.0:1775767998.129643:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000008:00000001:3.0:1775767998.129645:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767998.129646:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767998.129646:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775767998.129649:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00000001:3.0:1775767998.129650:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767998.129651:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767998.129654:0:13471:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775767998.129654:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aa300. 00000400:00000200:0.0:1775767998.129656:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:2.0:1775767998.129658:0:13471:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107621738216 : -112966087813400 : ffff99420876d6e8) 00000020:00000040:3.0:1775767998.129660:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:2.0:1775767998.129663:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000400:00000010:0.0:1775767998.129664:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 02000000:00000001:3.0:1775767998.129664:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000020:00000001:2.0:1775767998.129665:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767998.129666:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767998.129670:0:13471:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767998.129670:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767998.129672:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000020:00000001:2.0:1775767998.129673:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775767998.129674:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000020:00000001:2.0:1775767998.129675:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 02000000:00000001:3.0:1775767998.129675:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767998.129678:0:13471:0:(lov_io.c:1108:lov_io_lock()) Process entered 00020000:00000001:2.0:1775767998.129680:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775767998.129681:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 02000000:00000010:3.0:1775767998.129682:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2c00. 00000020:00000001:2.0:1775767998.129683:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000100:00000200:0.0:1775767998.129684:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294261120, portal 25 00000020:00000001:2.0:1775767998.129685:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767998.129687:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767998.129690:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000020:00000001:2.0:1775767998.129691:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767998.129693:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640855168 : -112966068696448 : ffff9942099a8a80) 00020000:00000001:2.0:1775767998.129694:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775767998.129695:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aa300 x1862022294261248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00020000:00000001:2.0:1775767998.129699:0:13471:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767998.129703:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a8a80 x1862022294261120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768023 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1775767998.129704:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:2.0:1775767998.129707:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767998.129712:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767998.129717:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00000100:00000001:3.0:1775767998.129717:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00020000:00000001:2.0:1775767998.129720:0:13471:0:(lov_io.c:1114:lov_io_start()) Process entered 00020000:00000001:2.0:1775767998.129723:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000001:3.0:1775767998.129723:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767998.129725:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:2.0:1775767998.129726:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00000008:00000001:2.0:1775767998.129729:0:13471:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000100:00000040:0.0:1775767998.129732:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000100:00000040:3.0:1775767998.129732:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f33bdf15] to pc [ptlrpcd_rcv+-1] req@ffff9942099aa300 x1862022294261248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:2.0:1775767998.129733:0:13471:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000400:00000010:0.0:1775767998.129735:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000008:00000001:2.0:1775767998.129738:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000100:00000200:0.0:1775767998.129740:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294261120, offset 0 00000008:00000020:2.0:1775767998.129741:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:2.0:1775767998.129745:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1775767998.129749:0:13471:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000008:00000001:2.0:1775767998.129752:0:13471:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000400:00000200:0.0:1775767998.129753:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00000001:3.0:1775767998.129753:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1775767998.129755:0:13471:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107884691392 : -112965824860224 : ffff994218232fc0) 00000100:00080000:3.0:1775767998.129757:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000008:00000020:2.0:1775767998.129761:0:13471:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994218232fc0 ready 0|-|- wr 1|-|+ rd 0|- 10 in flight 00000100:00080000:3.0:1775767998.129762:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775767998.129765:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:0.0:1775767998.129768:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00000001:2.0:1775767998.129768:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000100:00080000:3.0:1775767998.129769:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000008:00000020:2.0:1775767998.129772:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000100:00000001:3.0:1775767998.129773:0:8:0:(import.c:484:import_select_connection()) Process entered 00000008:00000001:2.0:1775767998.129774:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1775767998.129778:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000008:00000001:2.0:1775767998.129778:0:13471:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000100:00080000:3.0:1775767998.129779:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5250 00000400:00000200:3.0:1775767998.129787:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767998.129789:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000008:00000020:2.0:1775767998.129789:0:13471:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994210675880 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000400:00000200:3.0:1775767998.129792:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775767998.129797:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000400:00000200:0.0:1775767998.129798:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000008:00000001:2.0:1775767998.129798:0:13471:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1775767998.129802:0:13471:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767998.129805:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000001:2.0:1775767998.129806:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767998.129807:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1775767998.129809:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00020000:00000001:2.0:1775767998.129810:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767998.129813:0:13471:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767998.129815:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000020:00000001:2.0:1775767998.129816:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767998.129820:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00000100:00000040:3.0:1775767998.129821:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00020000:00000001:2.0:1775767998.129823:0:13471:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00000100:00000001:3.0:1775767998.129825:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000200:0.0:1775767998.129827:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225213:225213:256:4294967295] 192.168.202.156@tcp LPNI seq info [225209:225209:8:4294967295] 00020000:00000001:2.0:1775767998.129827:0:13471:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000020:00000001:3.0:1775767998.129828:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000001:2.0:1775767998.129830:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000040:3.0:1775767998.129831:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:2.0:1775767998.129833:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00000020:00000001:3.0:1775767998.129834:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00020000:00000001:2.0:1775767998.129836:0:13471:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775767998.129837:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00020000:00000001:2.0:1775767998.129838:0:13471:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000400:00000200:0.0:1775767998.129842:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:2.0:1775767998.129842:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00000020:00000001:3.0:1775767998.129842:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:2.0:1775767998.129845:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00020000:00000001:2.0:1775767998.129848:0:13471:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00000020:00000001:3.0:1775767998.129848:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00020000:00000001:2.0:1775767998.129850:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775767998.129853:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000100:00000040:3.0:1775767998.129854:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000001:2.0:1775767998.129856:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:2.0:1775767998.129860:0:13471:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767998.129860:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1775767998.129861:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00020000:00000001:2.0:1775767998.129865:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775767998.129865:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00020000:00000001:2.0:1775767998.129871:0:13471:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000100:00000001:3.0:1775767998.129871:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775767998.129873:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:2.0:1775767998.129876:0:13471:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000020:00000040:3.0:1775767998.129876:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000800:00000200:0.0:1775767998.129877:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000020:00000001:2.0:1775767998.129880:0:13471:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000001:2.0:1775767998.129883:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000100:00080000:3.0:1775767998.129883:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000800:00000010:0.0:1775767998.129884:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00020000:00000001:2.0:1775767998.129886:0:13471:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00020000:00000001:2.0:1775767998.129889:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000001:3.0:1775767998.129890:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767998.129891:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 02000000:00000001:3.0:1775767998.129892:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000020:00000001:2.0:1775767998.129893:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00020000:00000001:2.0:1775767998.129896:0:13471:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1775767998.129897:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 02000000:00000001:3.0:1775767998.129897:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:2.0:1775767998.129899:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767998.129903:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1775767998.129904:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00020000:00000001:2.0:1775767998.129904:0:13471:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000020:00000001:2.0:1775767998.129907:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000100:00000001:3.0:1775767998.129907:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000020:00000001:2.0:1775767998.129910:0:13471:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1775767998.129911:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775767998.129927:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775767998.129928:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:3.0:1775767998.129929:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775767998.129934:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000100:00000010:3.0:1775767998.129935:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ab800. 00000020:00000001:2.0:1775767998.129937:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:2.0:1775767998.129940:0:13471:0:(lov_io.c:675:lov_io_fini()) Process entered 00000020:00000040:3.0:1775767998.129941:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00020000:00000001:2.0:1775767998.129943:0:13471:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000800:00000100:0.0:1775767998.129945:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000020:00000001:2.0:1775767998.129946:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 02000000:00000001:3.0:1775767998.129947:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000800:00000010:0.0:1775767998.129949:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000020:00000001:2.0:1775767998.129949:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 02000000:00000001:3.0:1775767998.129950:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:2.0:1775767998.129952:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff994210b32090 00000100:00000001:3.0:1775767998.129953:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775767998.129955:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:0.0:1775767998.129957:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000010:2.0:1775767998.129957:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994209d2cc00. 02000000:00000001:3.0:1775767998.129957:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775767998.129965:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0000. 00000400:00000200:0.0:1775767998.129966:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000020:00000010:2.0:1775767998.129966:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942182d6480. 00000100:00000001:3.0:1775767998.129970:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:2.0:1775767998.129973:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207cc8b28. 00000400:00000200:0.0:1775767998.129974:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775767998.129978:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00100000:3.0:1775767998.129978:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ab800 x1862022294261376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00020000:00000010:2.0:1775767998.129979:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994220156840. 00000100:00000001:0.0:1775767998.129985:0:2399:0:(events.c:34:request_out_callback()) Process entered 00008000:00000010:2.0:1775767998.129986:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942077a9530. 00000100:00000200:0.0:1775767998.129992:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a8a80 x1862022294261120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768023 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000010:2.0:1775767998.129992:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994208747738. 00000100:00000001:3.0:1775767998.129992:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000080:00000010:2.0:1775767998.129997:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077ba910. 00000100:00000001:3.0:1775767998.129997:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1775767998.130003:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994209d2dc00. 00000100:00000040:3.0:1775767998.130007:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000089df80c0] to pc [ptlrpcd_rcv+-1] req@ffff9942099ab800 x1862022294261376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767998.130010:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000008:00000010:2.0:1775767998.130010:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942077cd8c0. 00020000:00000010:2.0:1775767998.130016:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff99420876e400. 00000100:00000040:0.0:1775767998.130017:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a8a80 x1862022294261120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768023 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00008000:00000010:2.0:1775767998.130021:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076c3f0. 00000100:00000001:3.0:1775767998.130026:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:2.0:1775767998.130027:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942050663c0. 00000100:00080000:3.0:1775767998.130032:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000010:2.0:1775767998.130033:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9942060d5800. 00000100:00000001:0.0:1775767998.130034:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767998.130040:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000010:2.0:1775767998.130040:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994210b32090. 00000100:00080000:3.0:1775767998.130040:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775767998.130044:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1775767998.130045:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767998.130046:0:13471:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00000100:00080000:3.0:1775767998.130046:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00020000:00000001:2.0:1775767998.130051:0:13471:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000100:00000001:3.0:1775767998.130051:0:8:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:0.0:1775767998.130054:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000080:00000001:2.0:1775767998.130055:0:13471:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000100:00080000:3.0:1775767998.130058:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5250 00000080:00200000:2.0:1775767998.130059:0:13471:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00000001:0.0:1775767998.130060:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299924285 00000400:00000200:3.0:1775767998.130066:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000010:0.0:1775767998.130067:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942078a3d80. 00000400:00000200:3.0:1775767998.130071:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000080:00000001:2.0:1775767998.130073:0:13471:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000100:00080000:3.0:1775767998.130074:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1775767998.130076:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000020:00000001:2.0:1775767998.130076:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000100:00000040:3.0:1775767998.130079:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00001000:2.0:1775767998.130081:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff994210b32630 00000100:00000001:3.0:1775767998.130082:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1775767998.130084:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299924285 00000020:00000010:2.0:1775767998.130085:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994209d2de00. 00000100:00000040:3.0:1775767998.130085:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000010:2.0:1775767998.130090:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942182d6300. 00000100:00000001:3.0:1775767998.130091:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000010:0.0:1775767998.130092:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942078a3ea0. 00000008:00000010:2.0:1775767998.130093:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207ccc2f0. 00000020:00000001:3.0:1775767998.130094:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767998.130096:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:3.0:1775767998.130098:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00020000:00000010:2.0:1775767998.130099:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994220156e40. 00000100:00000001:0.0:1775767998.130100:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000020:00000040:3.0:1775767998.130100:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00008000:00000010:2.0:1775767998.130103:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942077a8e20. 00000020:00000001:3.0:1775767998.130104:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:0.0:1775767998.130107:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000010:2.0:1775767998.130108:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff9942087472a0. 00000020:00000001:3.0:1775767998.130108:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:3.0:1775767998.130114:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775767998.130115:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:2.0:1775767998.130115:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b8000. 00000100:00000001:3.0:1775767998.130118:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000010:2.0:1775767998.130120:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994209d2d600. 00000100:00000001:0.0:1775767998.130121:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000040:3.0:1775767998.130122:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000008:00000010:2.0:1775767998.130125:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942077cd600. 00000100:00000001:3.0:1775767998.130126:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00020000:00000010:2.0:1775767998.130130:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff99420876d680. 00000020:00000040:3.0:1775767998.130131:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00100000:0.0:1775767998.130132:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767998/real 1775767998] req@ffff9942099a8a80 x1862022294261120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00008000:00000010:2.0:1775767998.130134:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076cf60. 00000100:00080000:3.0:1775767998.130137:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000080:00000010:2.0:1775767998.130139:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff994205066870. 00000100:00000001:3.0:1775767998.130142:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1775767998.130145:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994209d2d000. 02000000:00000001:3.0:1775767998.130147:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000020:00000010:2.0:1775767998.130149:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994210b32630. 02000000:00000001:3.0:1775767998.130151:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000001:2.0:1775767998.130174:0:13471:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1775767998.130175:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775767998.130177:0:13471:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775767998.130178:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775767998.130181:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000080:00000001:2.0:1775767998.130184:0:13471:0:(rw.c:1481:ll_writepages()) Process entered 00000400:00000001:0.0:1775767998.130185:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000008:00000001:3.0:1775767998.130185:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775767998.130187:0:13471:0:(file.c:5250:cl_sync_file_range()) Process entered 00000400:00000001:0.0:1775767998.130189:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1775767998.130189:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767998.130190:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000400:00000010:0.0:1775767998.130193:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000020:00000010:2.0:1775767998.130195:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994210b32630. 00000100:00000010:3.0:1775767998.130195:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aa680. 00000400:00000200:0.0:1775767998.130198:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000020:00000040:3.0:1775767998.130199:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000010:2.0:1775767998.130200:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9942060d5800. 00000400:00000010:0.0:1775767998.130201:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 02000000:00000001:3.0:1775767998.130203:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000020:00000010:2.0:1775767998.130206:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff9942060d5a00. 02000000:00000001:3.0:1775767998.130206:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767998.130207:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1775767998.130211:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000020:00000010:2.0:1775767998.130212:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942182d6300. 00000100:00000200:0.0:1775767998.130214:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a8a80 x1862022294261120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1775767998.130215:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000008:00000010:2.0:1775767998.130216:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207ccc2f0. 02000000:00000001:3.0:1775767998.130218:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:2.0:1775767998.130221:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994220156e40. 02000000:00000010:3.0:1775767998.130224:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0400. 00008000:00000010:2.0:1775767998.130227:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942077a8e20. 00000100:00000001:3.0:1775767998.130230:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:2.0:1775767998.130232:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff9942087472a0. 00000080:00000010:2.0:1775767998.130238:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b8000. 00000020:00000010:2.0:1775767998.130241:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9942060d4c00. 00000100:00000200:0.0:1775767998.130243:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a8a80 x1862022294261120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767998.130244:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aa680 x1862022294261504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000008:00000010:2.0:1775767998.130245:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942077cd600. 00020000:00000010:2.0:1775767998.130250:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff99420876d680. 00008000:00000010:2.0:1775767998.130255:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076cf60. 00000080:00000010:2.0:1775767998.130261:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff994205066870. 00000020:00000001:2.0:1775767998.130267:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107759908408 : -112965949643208 : ffff994210b32638) 00000100:00000001:0.0:1775767998.130268:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775767998.130270:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00001000:2.0:1775767998.130274:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff994210b32630 00000100:00000001:0.0:1775767998.130275:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775767998.130275:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000020:00000001:2.0:1775767998.130276:0:13471:0:(cl_object.c:151:cl_object_top()) ffff9942078a7f58 -> ffff9942078a7f58 00000100:00000001:0.0:1775767998.130278:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1775767998.130280:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000100:00000001:3.0:1775767998.130281:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775767998.130282:0:13471:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000100:00100000:0.0:1775767998.130285:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a8a80 x1862022294261120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00200000:2.0:1775767998.130286:0:13471:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017b:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:2.0:1775767998.130293:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000100:00000040:3.0:1775767998.130293:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006cb1597d] to pc [ptlrpcd_rcv+-1] req@ffff9942099aa680 x1862022294261504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1775767998.130296:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:2.0:1775767998.130299:0:13471:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775767998.130303:0:13471:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017b:0x0]io ffff9942077b8060 type 6 ignore/verify layout 1/0 00020000:00000002:2.0:1775767998.130310:0:13471:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207c1f000) owner 0000000000000000/ffff994220080000 00000100:00000001:0.0:1775767998.130311:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00020000:00000001:2.0:1775767998.130316:0:13471:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:2.0:1775767998.130318:0:13471:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00000100:00000001:3.0:1775767998.130318:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767998.130321:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a8a80 x1862022294261120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768023 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:2.0:1775767998.130322:0:13471:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00000100:00080000:3.0:1775767998.130323:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00020000:00000001:2.0:1775767998.130325:0:13471:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767998.130330:0:13471:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1775767998.130332:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00020000:00000001:2.0:1775767998.130336:0:13471:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00000100:00000001:3.0:1775767998.130337:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00020000:00000001:2.0:1775767998.130340:0:13471:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775767998.130340:0:8:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775767998.130341:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:2.0:1775767998.130346:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000100:00000001:3.0:1775767998.130347:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:0.0:1775767998.130348:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1775767998.130348:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00000001:0.0:1775767998.130352:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00020000:00000001:2.0:1775767998.130352:0:13471:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00000100:00080000:3.0:1775767998.130353:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5250 00020000:00000002:2.0:1775767998.130354:0:13471:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207c1f000) owner 0000000000000000/ffff994220080000 00000100:00000001:0.0:1775767998.130355:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1775767998.130359:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767998.130361:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000400:00000200:3.0:1775767998.130362:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:2.0:1775767998.130364:0:13471:0:(cl_io.c:823:cl_io_loop()) Process entered 00000100:00000001:0.0:1775767998.130366:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:2.0:1775767998.130366:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00020000:00000001:2.0:1775767998.130369:0:13471:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00000400:00000200:3.0:1775767998.130369:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775767998.130373:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00020000:00200000:2.0:1775767998.130373:0:13471:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00080000:3.0:1775767998.130375:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00020000:00000002:2.0:1775767998.130378:0:13471:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:2.0:1775767998.130381:0:13471:0:(lov_io.c:155:lov_sub_get()) Process entered 00000100:00080000:0.0:1775767998.130384:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00020000:00000001:2.0:1775767998.130384:0:13471:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000100:00000040:3.0:1775767998.130384:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000001:2.0:1775767998.130385:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775767998.130388:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994210b32090. 00000100:00000001:0.0:1775767998.130391:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000010:2.0:1775767998.130392:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9942060d5200. 00000100:00000001:3.0:1775767998.130392:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000010:2.0:1775767998.130398:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff9942060d4800. 00000100:00000040:3.0:1775767998.130398:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000010:2.0:1775767998.130400:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942182d6480. 00000100:00000040:0.0:1775767998.130402:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a8a80 x1862022294261120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768023 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775767998.130404:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000008:00000010:2.0:1775767998.130405:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207cc8b28. 00020000:00000010:2.0:1775767998.130409:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994220156840. 00000020:00000001:3.0:1775767998.130411:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775767998.130414:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00008000:00000010:2.0:1775767998.130415:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942077a9530. 00000020:00000001:3.0:1775767998.130419:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000080:00000010:2.0:1775767998.130420:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994208747738. 00000020:00000040:3.0:1775767998.130424:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000080:00000010:2.0:1775767998.130425:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077ba910. 00000100:00100000:0.0:1775767998.130429:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294261120:192.168.202.156@tcp:250:kworker.0 00000020:00000010:2.0:1775767998.130431:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994209d2d000. 00000020:00000001:3.0:1775767998.130431:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000008:00000010:2.0:1775767998.130437:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942077cd8c0. 00000100:00000001:0.0:1775767998.130438:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1775767998.130438:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00020000:00000010:2.0:1775767998.130440:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff99420876e400. 00000100:00000001:0.0:1775767998.130442:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00008000:00000010:2.0:1775767998.130446:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076c3f0. 00000100:00000040:0.0:1775767998.130449:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a8a80 x1862022294261120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768023 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000010:2.0:1775767998.130450:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942050663c0. 00000100:00000040:3.0:1775767998.130450:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000001:2.0:1775767998.130455:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107759906968 : -112965949644648 : ffff994210b32098) 00000100:00000001:3.0:1775767998.130457:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00001000:2.0:1775767998.130461:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff994210b32090 00000100:00000040:3.0:1775767998.130463:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000001:2.0:1775767998.130466:0:13471:0:(cl_object.c:151:cl_object_top()) ffff9942078a75f8 -> ffff9942078a7f58 00000100:00000001:0.0:1775767998.130467:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1775767998.130469:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775767998.130470:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 02000000:00000001:0.0:1775767998.130471:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000001:2.0:1775767998.130474:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 02000000:00000010:0.0:1775767998.130475:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002bc00. 00000020:00000040:3.0:1775767998.130475:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000020:00000001:2.0:1775767998.130477:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:2.0:1775767998.130481:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767998.130482:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00080000:3.0:1775767998.130483:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000020:00000001:0.0:1775767998.130485:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00020000:00000001:2.0:1775767998.130487:0:13471:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1775767998.130488:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000100:00000001:3.0:1775767998.130489:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767998.130493:0:13471:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107621738216 : -112966087813400 : ffff99420876d6e8) 00000020:00000001:0.0:1775767998.130494:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000001:3.0:1775767998.130494:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000010:0.0:1775767998.130498:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b2800. 00000020:00000001:2.0:1775767998.130499:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 02000000:00000001:3.0:1775767998.130499:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775767998.130503:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000020:00000001:2.0:1775767998.130503:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767998.130504:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767998.130505:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775767998.130508:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a8a80. 00000100:00000001:3.0:1775767998.130509:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00020000:00000001:2.0:1775767998.130510:0:13471:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767998.130513:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000008:00100000:3.0:1775767998.130513:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000020:00000001:2.0:1775767998.130514:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767998.130516:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1775767998.130520:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00000100:00000001:0.0:1775767998.130521:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1775767998.130521:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767998.130523:0:13471:0:(lov_io.c:1108:lov_io_lock()) Process entered 00020000:00000001:2.0:1775767998.130527:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000001:3.0:1775767998.130527:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767998.130529:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767998.130531:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767998.130531:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00000100:00000010:3.0:1775767998.130533:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aad80. 00000020:00000001:2.0:1775767998.130534:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000100:00000001:0.0:1775767998.130535:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1775767998.130537:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767998.130538:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000040:3.0:1775767998.130539:0:8:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00000100:00000001:0.0:1775767998.130540:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000001:2.0:1775767998.130542:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767998.130546:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767998.130547:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099aa300 x1862022294261248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1775767998.130547:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00020000:00000001:2.0:1775767998.130550:0:13471:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767998.130551:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767998.130556:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000100:00000001:3.0:1775767998.130558:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000020:00000001:2.0:1775767998.130559:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775767998.130562:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000020:00000001:2.0:1775767998.130564:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767998.130567:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000001:2.0:1775767998.130567:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 02000000:00000001:3.0:1775767998.130567:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767998.130569:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767998.130570:0:13471:0:(lov_io.c:1114:lov_io_start()) Process entered 00020000:00000001:2.0:1775767998.130573:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 02000000:00000001:0.0:1775767998.130574:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:2.0:1775767998.130574:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 02000000:00000001:0.0:1775767998.130577:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:2.0:1775767998.130577:0:13471:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 02000000:00000010:3.0:1775767998.130577:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b3400. 00000008:00000001:2.0:1775767998.130580:0:13471:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 02000000:00000001:0.0:1775767998.130582:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1775767998.130585:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000100:00000001:3.0:1775767998.130585:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767998.130586:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099aa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294261248:192.168.202.156@tcp:38:kworker.0 00000008:00000020:2.0:1775767998.130588:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:2.0:1775767998.130590:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767998.130593:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000008:00000001:2.0:1775767998.130594:0:13471:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 02000000:00000001:0.0:1775767998.130597:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000008:00000001:2.0:1775767998.130598:0:13471:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000100:00100000:3.0:1775767998.130598:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aad80 x1862022294261632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1775767998.130599:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1775767998.130600:0:13471:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107884688256 : -112965824863360 : ffff994218232380) 02000000:00000001:0.0:1775767998.130605:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000020:2.0:1775767998.130606:0:13471:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994218232380 ready 0|-|- wr 1|-|+ rd 0|- 9 in flight 02000000:00000010:0.0:1775767998.130608:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002bc00. 02000000:00000001:0.0:1775767998.130613:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1775767998.130614:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:2.0:1775767998.130618:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:2.0:1775767998.130620:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1775767998.130623:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00000001:2.0:1775767998.130624:0:13471:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000400:00000010:0.0:1775767998.130627:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000008:00000020:2.0:1775767998.130628:0:13471:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994210674fc0 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000100:00000001:3.0:1775767998.130628:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000010:0.0:1775767998.130634:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000001:3.0:1775767998.130634:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1775767998.130635:0:13471:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:0.0:1775767998.130639:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294261248, portal 10 00000008:00000001:2.0:1775767998.130641:0:13471:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767998.130642:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000020:00000001:2.0:1775767998.130644:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767998.130645:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640861440 : -112966068690176 : ffff9942099aa300) 00000100:00000040:3.0:1775767998.130646:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000045ccf08] to pc [ptlrpcd_rcv+-1] req@ffff9942099aad80 x1862022294261632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:2.0:1775767998.130648:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767998.130652:0:13471:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767998.130655:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099aa300 x1862022294261248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768023 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1775767998.130656:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767998.130661:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00020000:00000001:2.0:1775767998.130663:0:13471:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00020000:00000001:2.0:1775767998.130666:0:13471:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000020:00000001:2.0:1775767998.130669:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000001:2.0:1775767998.130671:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00020000:00000001:2.0:1775767998.130674:0:13471:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767998.130678:0:13471:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000100:00000001:0.0:1775767998.130681:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1775767998.130681:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767998.130682:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00000020:00000001:2.0:1775767998.130685:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00020000:00000001:2.0:1775767998.130687:0:13471:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00000100:00000040:0.0:1775767998.130688:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000100:00000040:3.0:1775767998.130689:0:8:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5260) 00020000:00000001:2.0:1775767998.130691:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000400:00000010:0.0:1775767998.130692:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000020:00000001:2.0:1775767998.130693:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000100:00000200:0.0:1775767998.130697:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294261248, offset 0 00000020:00000001:2.0:1775767998.130697:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:2.0:1775767998.130700:0:13471:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767998.130703:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00020000:00000001:2.0:1775767998.130703:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767998.130708:0:13471:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000020:00000001:2.0:1775767998.130709:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000400:00000200:0.0:1775767998.130710:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:2.0:1775767998.130711:0:13471:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000020:00000001:2.0:1775767998.130714:0:13471:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000001:2.0:1775767998.130715:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000400:00000200:0.0:1775767998.130716:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00020000:00000001:2.0:1775767998.130717:0:13471:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00020000:00000001:2.0:1775767998.130721:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000400:00000200:0.0:1775767998.130722:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:2.0:1775767998.130724:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000400:00000200:0.0:1775767998.130727:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000020:00000001:2.0:1775767998.130727:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00020000:00000001:2.0:1775767998.130730:0:13471:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767998.130731:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00020000:00000001:2.0:1775767998.130734:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767998.130735:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00020000:00000001:2.0:1775767998.130737:0:13471:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000400:00000200:0.0:1775767998.130740:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225214:225214:256:4294967295] 192.168.202.156@tcp LPNI seq info [225210:225210:8:4294967295] 00000020:00000001:2.0:1775767998.130740:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000020:00000001:2.0:1775767998.130742:0:13471:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767998.130746:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:2.0:1775767998.130748:0:13471:0:(lov_io.c:675:lov_io_fini()) Process entered 00000400:00000200:0.0:1775767998.130750:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:2.0:1775767998.130750:0:13471:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:2.0:1775767998.130753:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:2.0:1775767998.130755:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:2.0:1775767998.130756:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff994210b32090 00000020:00000010:2.0:1775767998.130759:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff9942060d4800. 00000020:00000010:2.0:1775767998.130764:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942182d6480. 00000400:00000200:0.0:1775767998.130765:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000008:00000010:2.0:1775767998.130767:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207cc8b28. 00020000:00000010:2.0:1775767998.130772:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994220156840. 00000800:00000200:0.0:1775767998.130775:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00008000:00000010:2.0:1775767998.130776:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942077a9530. 00000080:00000010:2.0:1775767998.130781:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994208747738. 00000800:00000010:0.0:1775767998.130782:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000080:00000010:2.0:1775767998.130782:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077ba910. 00000020:00000010:2.0:1775767998.130787:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9942060d5200. 00000008:00000010:2.0:1775767998.130790:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942077cd8c0. 00000800:00000200:0.0:1775767998.130793:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00020000:00000010:2.0:1775767998.130794:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff99420876e400. 00008000:00000010:2.0:1775767998.130798:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076c3f0. 00000800:00000200:0.0:1775767998.130799:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000080:00000010:2.0:1775767998.130801:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942050663c0. 00000020:00000010:2.0:1775767998.130805:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994209d2d000. 00000020:00000010:2.0:1775767998.130808:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994210b32090. 00000800:00000200:0.1:1775767998.130810:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00020000:00000001:2.0:1775767998.130811:0:13471:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:2.0:1775767998.130814:0:13471:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:2.0:1775767998.130816:0:13471:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000800:00000200:0.1:1775767998.130816:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000080:00200000:2.0:1775767998.130819:0:13471:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:2.0:1775767998.130823:0:13471:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000800:00000100:0.0:1775767998.130825:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000020:00000001:2.0:1775767998.130825:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:2.0:1775767998.130828:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff994210b32630 00000800:00000010:0.0:1775767998.130829:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000020:00000010:2.0:1775767998.130830:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff9942060d5a00. 00000020:00000010:2.0:1775767998.130834:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942182d6300. 00000400:00000200:0.0:1775767998.130836:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000008:00000010:2.0:1775767998.130837:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207ccc2f0. 00020000:00000010:2.0:1775767998.130841:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994220156e40. 00000400:00000200:0.0:1775767998.130844:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00008000:00000010:2.0:1775767998.130846:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942077a8e20. 00000400:00000200:0.0:1775767998.130848:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775767998.130851:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000080:00000010:2.0:1775767998.130851:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff9942087472a0. 00000100:00000001:0.0:1775767998.130854:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000080:00000010:2.0:1775767998.130856:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b8000. 00000100:00000200:0.0:1775767998.130859:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099aa300 x1862022294261248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768023 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1775767998.130860:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9942060d5800. 00000008:00000010:2.0:1775767998.130865:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942077cd600. 00020000:00000010:2.0:1775767998.130869:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff99420876d680. 00008000:00000010:2.0:1775767998.130873:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076cf60. 00000100:00000001:0.0:1775767998.130875:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000080:00000010:2.0:1775767998.130876:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff994205066870. 00000100:00000040:0.0:1775767998.130879:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099aa300 x1862022294261248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768023 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1775767998.130880:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9942060d4c00. 00000020:00000010:2.0:1775767998.130885:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994210b32630. 00000080:00000001:2.0:1775767998.130889:0:13471:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:2.0:1775767998.130894:0:13471:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767998.130895:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767998.130897:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000080:00000001:2.0:1775767998.130899:0:13471:0:(rw.c:1481:ll_writepages()) Process entered 00000100:00000001:0.0:1775767998.130901:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775767998.130902:0:13471:0:(file.c:5250:cl_sync_file_range()) Process entered 00000020:00000001:2.0:1775767998.130905:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00000040:0.0:1775767998.130906:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000020:00000010:2.0:1775767998.130907:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994210b32630. 00000020:00000010:2.0:1775767998.130924:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994209d2d000. 00000400:00000200:0.0:1775767998.130925:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000010:2.0:1775767998.130928:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994209d2d600. 00000100:00000001:0.0:1775767998.130929:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299922963 00000020:00000010:2.0:1775767998.130931:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942182d6300. 00000100:00000001:0.0:1775767998.130933:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299924285 00000008:00000010:2.0:1775767998.130935:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207ccc2f0. 00000100:00000001:0.0:1775767998.130936:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299922963 00020000:00000010:2.0:1775767998.130939:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994220156e40. 00000100:00000001:0.0:1775767998.130940:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299924285 00008000:00000010:2.0:1775767998.130943:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942077a8e20. 00000100:00000001:0.0:1775767998.130944:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775767998.130948:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:2.0:1775767998.130948:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff9942087472a0. 00000100:00000001:0.0:1775767998.130950:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000080:00000010:2.0:1775767998.130952:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b8000. 00000100:00100000:0.0:1775767998.130955:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767998/real 1775767998] req@ffff9942099aa300 x1862022294261248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1775767998.130956:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994209d2de00. 00000008:00000010:2.0:1775767998.130960:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942077cd600. 00020000:00000010:2.0:1775767998.130965:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff99420876d680. 00008000:00000010:2.0:1775767998.130970:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076cf60. 00000400:00000001:0.0:1775767998.130971:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000080:00000010:2.0:1775767998.130972:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff994205066870. 00000400:00000001:0.0:1775767998.130973:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:2.0:1775767998.130976:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107759908408 : -112965949643208 : ffff994210b32638) 00000400:00000010:0.0:1775767998.130977:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775767998.130981:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000020:00001000:2.0:1775767998.130981:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff994210b32630 00000400:00000010:0.0:1775767998.130984:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000020:00000001:2.0:1775767998.130985:0:13471:0:(cl_object.c:151:cl_object_top()) ffff99420791a850 -> ffff99420791a850 00000100:00000001:0.0:1775767998.130986:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:2.0:1775767998.130988:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000100:00000200:0.0:1775767998.130991:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099aa300 x1862022294261248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000001:2.0:1775767998.130992:0:13471:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:2.0:1775767998.130995:0:13471:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017f:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:2.0:1775767998.131001:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775767998.131004:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:2.0:1775767998.131005:0:13471:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1775767998.131006:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099aa300 x1862022294261248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000002:2.0:1775767998.131008:0:13471:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017f:0x0]io ffff9942077b8060 type 6 ignore/verify layout 1/0 00020000:00000002:2.0:1775767998.131016:0:13471:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207dfbd90) owner 0000000000000000/ffff994220080000 00000100:00000001:0.0:1775767998.131020:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00020000:00000001:2.0:1775767998.131021:0:13471:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00000100:00000001:0.0:1775767998.131023:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00020000:00000001:2.0:1775767998.131023:0:13471:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00000100:00000001:0.0:1775767998.131026:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00020000:00000001:2.0:1775767998.131026:0:13471:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00020000:00000001:2.0:1775767998.131029:0:13471:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767998.131030:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00020000:00000001:2.0:1775767998.131034:0:13471:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1775767998.131035:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099aa300 x1862022294261248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:2.0:1775767998.131036:0:13471:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:2.0:1775767998.131039:0:13471:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767998.131043:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775767998.131045:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:2.0:1775767998.131047:0:13471:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00000100:00000001:0.0:1775767998.131048:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00020000:00000002:2.0:1775767998.131049:0:13471:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207dfbd90) owner 0000000000000000/ffff994220080000 00000020:00000001:2.0:1775767998.131052:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767998.131053:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099aa300 x1862022294261248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768023 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1775767998.131054:0:13471:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:2.0:1775767998.131056:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00020000:00000001:2.0:1775767998.131058:0:13471:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00020000:00200000:2.0:1775767998.131060:0:13471:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00000002:2.0:1775767998.131064:0:13471:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00000100:00000001:0.0:1775767998.131065:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00020000:00000001:2.0:1775767998.131067:0:13471:0:(lov_io.c:155:lov_sub_get()) Process entered 00000100:00000001:0.0:1775767998.131069:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00020000:00000001:2.0:1775767998.131069:0:13471:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000100:00000001:0.0:1775767998.131072:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:2.0:1775767998.131072:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00000001:0.0:1775767998.131075:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:2.0:1775767998.131075:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994210b32090. 00000100:00000001:0.0:1775767998.131077:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000020:00000010:2.0:1775767998.131079:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994209d2dc00. 00000100:00000001:0.0:1775767998.131080:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000010:2.0:1775767998.131082:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994209d2cc00. 00000100:00080000:0.0:1775767998.131083:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000020:00000010:2.0:1775767998.131086:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942182d6480. 00000100:00080000:0.0:1775767998.131088:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775767998.131091:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000008:00000010:2.0:1775767998.131091:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207cc8b28. 00020000:00000010:2.0:1775767998.131096:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994220156840. 00000100:00000040:0.0:1775767998.131097:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099aa300 x1862022294261248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768023 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00008000:00000010:2.0:1775767998.131099:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942077a9530. 00000080:00000010:2.0:1775767998.131104:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994208747738. 00000080:00000010:2.0:1775767998.131108:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077ba910. 00000100:00100000:0.0:1775767998.131111:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099aa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294261248:192.168.202.156@tcp:38:kworker.0 00000020:00000010:2.0:1775767998.131112:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9942060d4c00. 00000008:00000010:2.0:1775767998.131117:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942077cd8c0. 00000100:00000001:0.0:1775767998.131118:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00020000:00000010:2.0:1775767998.131120:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff99420876e400. 00000100:00000040:0.0:1775767998.131124:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ab800 x1862022294261376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00008000:00000010:2.0:1775767998.131124:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076c3f0. 00000080:00000010:2.0:1775767998.131128:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942050663c0. 00000020:00000001:2.0:1775767998.131131:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107759906968 : -112965949644648 : ffff994210b32098) 00000020:00001000:2.0:1775767998.131136:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff994210b32090 00000020:00000001:2.0:1775767998.131138:0:13471:0:(cl_object.c:151:cl_object_top()) ffff99420791a210 -> ffff99420791a850 00000020:00000001:2.0:1775767998.131142:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:2.0:1775767998.131144:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775767998.131145:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:2.0:1775767998.131147:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767998.131151:0:13471:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767998.131175:0:13471:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107621738216 : -112966087813400 : ffff99420876d6e8) 00000100:00000001:0.0:1775767998.131177:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000001:2.0:1775767998.131178:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000100:00000001:0.0:1775767998.131180:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767998.131180:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767998.131186:0:13471:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767998.131187:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:2.0:1775767998.131188:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767998.131190:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1775767998.131190:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00020000:00000001:2.0:1775767998.131193:0:13471:0:(lov_io.c:1108:lov_io_lock()) Process entered 02000000:00000001:0.0:1775767998.131196:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767998.131196:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775767998.131198:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:2.0:1775767998.131200:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:2.0:1775767998.131202:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767998.131203:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294261376:192.168.202.156@tcp:38:kworker.0 00000020:00000001:2.0:1775767998.131204:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767998.131207:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767998.131210:0:13471:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767998.131213:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000001:2.0:1775767998.131213:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:2.0:1775767998.131216:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767998.131217:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:2.0:1775767998.131218:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767998.131220:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767998.131221:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00020000:00000001:2.0:1775767998.131222:0:13471:0:(lov_io.c:1114:lov_io_start()) Process entered 00020000:00000001:2.0:1775767998.131225:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 02000000:00000001:0.0:1775767998.131226:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:2.0:1775767998.131226:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00000008:00000001:2.0:1775767998.131229:0:13471:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000008:00000001:2.0:1775767998.131230:0:13471:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000008:00000001:2.0:1775767998.131231:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 02000000:00000010:0.0:1775767998.131233:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220028000. 00000008:00000020:2.0:1775767998.131233:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:2.0:1775767998.131234:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1775767998.131239:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1775767998.131239:0:13471:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000008:00000001:2.0:1775767998.131240:0:13471:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000008:00000001:2.0:1775767998.131243:0:13471:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107754938304 : -112965954613312 : ffff994210674fc0) 00000400:00000200:0.0:1775767998.131248:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00000020:2.0:1775767998.131248:0:13471:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994210674fc0 ready 0|-|- wr 1|-|+ rd 0|- 9 in flight 00000008:00000001:2.0:1775767998.131253:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:2.0:1775767998.131254:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000400:00000010:0.0:1775767998.131255:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000008:00000001:2.0:1775767998.131256:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1775767998.131260:0:13471:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000008:00000020:2.0:1775767998.131262:0:13471:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994218232380 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000400:00000010:0.0:1775767998.131263:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000008:00000001:2.0:1775767998.131266:0:13471:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:0.0:1775767998.131269:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294261376, portal 10 00000008:00000001:2.0:1775767998.131269:0:13471:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767998.131273:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767998.131274:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00020000:00000001:2.0:1775767998.131275:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767998.131277:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640866816 : -112966068684800 : ffff9942099ab800) 00020000:00000001:2.0:1775767998.131277:0:13471:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767998.131280:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775767998.131282:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00020000:00000001:2.0:1775767998.131283:0:13471:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00020000:00000001:2.0:1775767998.131285:0:13471:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000020:00000001:2.0:1775767998.131286:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000001:2.0:1775767998.131287:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00000100:00000040:0.0:1775767998.131288:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ab800 x1862022294261376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768023 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:2.0:1775767998.131289:0:13471:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767998.131292:0:13471:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000020:00000001:2.0:1775767998.131294:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00000020:00000001:2.0:1775767998.131296:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00020000:00000001:2.0:1775767998.131299:0:13471:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00020000:00000001:2.0:1775767998.131301:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775767998.131303:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000100:00000001:0.0:1775767998.131304:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:2.0:1775767998.131304:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:2.0:1775767998.131307:0:13471:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767998.131309:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00020000:00000001:2.0:1775767998.131311:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775767998.131312:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00020000:00000001:2.0:1775767998.131313:0:13471:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000100:00000200:0.0:1775767998.131316:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294261376, offset 0 00000020:00000001:2.0:1775767998.131316:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:2.0:1775767998.131318:0:13471:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000020:00000001:2.0:1775767998.131320:0:13471:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000001:2.0:1775767998.131322:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000400:00000200:0.0:1775767998.131323:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00020000:00000001:2.0:1775767998.131325:0:13471:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00020000:00000001:2.0:1775767998.131327:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775767998.131328:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000400:00000200:0.0:1775767998.131329:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:2.0:1775767998.131330:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00020000:00000001:2.0:1775767998.131332:0:13471:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767998.131335:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00020000:00000001:2.0:1775767998.131336:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775767998.131339:0:13471:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000400:00000200:0.0:1775767998.131340:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:2.0:1775767998.131341:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000020:00000001:2.0:1775767998.131343:0:13471:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767998.131344:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000020:00000001:2.0:1775767998.131347:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00000400:00000200:0.0:1775767998.131349:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00020000:00000001:2.0:1775767998.131350:0:13471:0:(lov_io.c:675:lov_io_fini()) Process entered 00000400:00000200:0.0:1775767998.131353:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00020000:00000001:2.0:1775767998.131353:0:13471:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:2.0:1775767998.131356:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:2.0:1775767998.131359:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000400:00000200:0.0:1775767998.131362:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225215:225215:256:4294967295] 192.168.202.156@tcp LPNI seq info [225211:225211:8:4294967295] 00000020:00001000:2.0:1775767998.131362:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff994210b32090 00000020:00000010:2.0:1775767998.131363:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994209d2cc00. 00000020:00000010:2.0:1775767998.131368:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942182d6480. 00000400:00000200:0.0:1775767998.131372:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00000010:2.0:1775767998.131372:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207cc8b28. 00020000:00000010:2.0:1775767998.131376:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994220156840. 00008000:00000010:2.0:1775767998.131378:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942077a9530. 00000080:00000010:2.0:1775767998.131382:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994208747738. 00000080:00000010:2.0:1775767998.131386:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077ba910. 00000400:00000200:0.0:1775767998.131387:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000020:00000010:2.0:1775767998.131389:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994209d2dc00. 00000008:00000010:2.0:1775767998.131392:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942077cd8c0. 00020000:00000010:2.0:1775767998.131396:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff99420876e400. 00000800:00000200:0.0:1775767998.131397:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00008000:00000010:2.0:1775767998.131399:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076c3f0. 00000080:00000010:2.0:1775767998.131402:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942050663c0. 00000800:00000010:0.0:1775767998.131403:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000020:00000010:2.0:1775767998.131405:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9942060d4c00. 00000020:00000010:2.0:1775767998.131408:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994210b32090. 00020000:00000001:2.0:1775767998.131412:0:13471:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00000800:00000200:0.0:1775767998.131413:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00020000:00000001:2.0:1775767998.131414:0:13471:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:2.0:1775767998.131416:0:13471:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000800:00000200:0.0:1775767998.131420:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000080:00200000:2.0:1775767998.131420:0:13471:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017f:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:2.0:1775767998.131426:0:13471:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:2.0:1775767998.131427:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:2.0:1775767998.131430:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff994210b32630 00000800:00000200:0.1:1775767998.131431:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000010:2.0:1775767998.131433:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994209d2d600. 00000800:00000200:0.1:1775767998.131435:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000020:00000010:2.0:1775767998.131438:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942182d6300. 00000800:00000100:0.0:1775767998.131441:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000008:00000010:2.0:1775767998.131442:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207ccc2f0. 00000800:00000010:0.0:1775767998.131443:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00020000:00000010:2.0:1775767998.131447:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994220156e40. 00000400:00000200:0.0:1775767998.131448:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00008000:00000010:2.0:1775767998.131450:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942077a8e20. 00000400:00000200:0.0:1775767998.131453:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000080:00000010:2.0:1775767998.131455:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff9942087472a0. 00000400:00000200:0.0:1775767998.131459:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000080:00000010:2.0:1775767998.131459:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b8000. 00000020:00000010:2.0:1775767998.131461:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994209d2d000. 00000400:00000010:0.0:1775767998.131463:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000008:00000010:2.0:1775767998.131464:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942077cd600. 00000100:00000001:0.0:1775767998.131469:0:2399:0:(events.c:34:request_out_callback()) Process entered 00020000:00000010:2.0:1775767998.131470:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff99420876d680. 00008000:00000010:2.0:1775767998.131474:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076cf60. 00000100:00000200:0.0:1775767998.131476:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ab800 x1862022294261376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768023 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000010:2.0:1775767998.131479:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff994205066870. 00000020:00000010:2.0:1775767998.131484:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994209d2de00. 00000020:00000010:2.0:1775767998.131489:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994210b32630. 00000100:00000001:0.0:1775767998.131494:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000080:00000001:2.0:1775767998.131494:0:13471:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:2.0:1775767998.131498:0:13471:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767998.131499:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ab800 x1862022294261376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768023 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767998.131508:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767998.131510:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775767998.131515:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767998.131518:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:0.0:1775767998.131523:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775767998.131525:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299922956 00000100:00000001:0.0:1775767998.131528:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299924285 00000100:00000001:0.0:1775767998.131530:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299922956 00000100:00000001:0.0:1775767998.131532:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299924285 00000100:00000001:0.0:1775767998.131535:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775767998.131537:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767998.131539:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775767998.131544:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767998/real 1775767998] req@ffff9942099ab800 x1862022294261376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775767998.131553:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775767998.131554:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775767998.131556:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775767998.131559:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775767998.131560:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775767998.131563:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775767998.131566:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ab800 x1862022294261376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775767998.131573:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ab800 x1862022294261376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767998.131577:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775767998.131578:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767998.131580:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767998.131580:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775767998.131582:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ab800 x1862022294261376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767998.131587:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775767998.131589:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ab800 x1862022294261376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768023 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767998.131593:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775767998.131595:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767998.131597:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767998.131598:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767998.131600:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775767998.131602:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775767998.131605:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767998.131609:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775767998.131611:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775767998.131615:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ab800 x1862022294261376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768023 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767998.131623:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294261376:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775767998.131627:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775767998.131631:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099aa680 x1862022294261504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767998.131638:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767998.131640:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767998.131642:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775767998.131644:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775767998.131646:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767998.131649:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099aa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294261504:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775767998.131653:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775767998.131654:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775767998.131656:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767998.131658:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775767998.131662:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220028400. 02000000:00000001:0.0:1775767998.131664:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767998.131668:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775767998.131671:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775767998.131674:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775767998.131678:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294261504, portal 4 00000100:00000001:0.0:1775767998.131680:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775767998.131681:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640862336 : -112966068689280 : ffff9942099aa680) 00000100:00000040:0.0:1775767998.131687:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099aa680 x1862022294261504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768023 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767998.131694:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775767998.131697:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775767998.131699:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775767998.131702:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294261504, offset 0 00000400:00000200:0.0:1775767998.131706:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775767998.131711:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767998.131715:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775767998.131719:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775767998.131721:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775767998.131724:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775767998.131726:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775767998.131730:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225216:225216:256:4294967295] 192.168.202.156@tcp LPNI seq info [225212:225212:8:4294967295] 00000400:00000200:0.0:1775767998.131734:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767998.131741:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775767998.131746:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767998.131750:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775767998.131754:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775767998.131757:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775767998.131762:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767998.131764:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000100:0.0:1775767998.131768:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767998.131770:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775767998.131773:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775767998.131777:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775767998.131780:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775767998.131782:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775767998.131785:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775767998.131789:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099aa680 x1862022294261504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768023 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767998.131797:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767998.131800:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099aa680 x1862022294261504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768023 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767998.131807:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767998.131808:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775767998.131812:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767998.131815:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:0.0:1775767998.131819:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775767998.131821:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299922979 00000100:00000001:0.0:1775767998.131823:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299924285 00000100:00000001:0.0:1775767998.131826:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299922979 00000100:00000001:0.0:1775767998.131829:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299924285 00000100:00000001:0.0:1775767998.131831:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775767998.131833:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767998.131835:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775767998.131839:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767998/real 1775767998] req@ffff9942099aa680 x1862022294261504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775767998.131848:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775767998.131849:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775767998.131851:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775767998.131854:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775767998.131855:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775767998.131857:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775767998.131861:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099aa680 x1862022294261504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775767998.131868:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099aa680 x1862022294261504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767998.131872:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775767998.131874:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767998.131875:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767998.131876:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775767998.131878:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099aa680 x1862022294261504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767998.131882:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775767998.131885:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099aa680 x1862022294261504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768023 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767998.131890:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775767998.131891:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767998.131892:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767998.131893:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767998.131894:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775767998.131897:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775767998.131898:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767998.131900:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775767998.131901:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775767998.131904:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099aa680 x1862022294261504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768023 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767998.131908:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099aa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294261504:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775767998.131923:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767998.131925:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767998.131927:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099aa300 x1862022294261248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768023 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767998.131932:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775767998.131933:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775767998.131935:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002bc00. 02000000:00000001:0.0:1775767998.131937:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775767998.131938:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775767998.131939:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:0.0:1775767998.131940:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775767998.131941:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b2c00. 02000000:00000001:0.0:1775767998.131943:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775767998.131944:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775767998.131945:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099aa300. 00000100:00000001:0.0:1775767998.131947:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775767998.131948:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767998.131949:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767998.131951:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ab800 x1862022294261376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768023 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767998.131955:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775767998.131956:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775767998.131957:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220028000. 02000000:00000001:0.0:1775767998.131958:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775767998.131958:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775767998.131959:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:0.0:1775767998.131960:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775767998.131961:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b0000. 02000000:00000001:0.0:1775767998.131963:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775767998.131963:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775767998.131964:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ab800. 00000100:00000001:0.0:1775767998.131966:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775767998.131966:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767998.131967:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767998.131969:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099aa680 x1862022294261504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768023 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767998.131972:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775767998.131973:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775767998.131974:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220028400. 02000000:00000001:0.0:1775767998.131975:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775767998.131976:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775767998.131976:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:0.0:1775767998.131977:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775767998.131978:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b0400. 02000000:00000001:0.0:1775767998.131979:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775767998.131980:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775767998.131981:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099aa680. 00000100:00000001:0.0:1775767998.131982:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775767998.131982:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767998.131983:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767998.131986:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767998.131987:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767998.131988:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775767998.131990:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767998.131991:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775767998.131993:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099aad80 x1862022294261632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767998.131998:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767998.131998:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767998.132000:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775767998.132001:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775767998.132002:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767998.132004:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099aad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294261632:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775767998.132006:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775767998.132008:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775767998.132009:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775767998.132010:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775767998.132012:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220028400. 02000000:00000001:0.0:1775767998.132013:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775767998.132016:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775767998.132017:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775767998.132020:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775767998.132022:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294261632, portal 4 00000100:00000001:0.0:1775767998.132023:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775767998.132024:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640864128 : -112966068687488 : ffff9942099aad80) 00000100:00000040:0.0:1775767998.132028:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099aad80 x1862022294261632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768023 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767998.132032:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775767998.132034:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775767998.132035:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775767998.132037:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294261632, offset 0 00000400:00000200:0.0:1775767998.132039:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775767998.132042:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767998.132044:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775767998.132046:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775767998.132048:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775767998.132050:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775767998.132052:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775767998.132054:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225217:225217:256:4294967295] 192.168.202.156@tcp LPNI seq info [225213:225213:8:4294967295] 00000400:00000200:0.0:1775767998.132056:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775767998.132061:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775767998.132063:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767998.132065:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775767998.132068:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775767998.132070:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775767998.132073:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775767998.132075:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000100:0.0:1775767998.132077:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775767998.132079:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775767998.132080:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775767998.132083:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775767998.132085:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775767998.132086:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775767998.132087:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775767998.132090:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099aad80 x1862022294261632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768023 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767998.132095:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767998.132097:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099aad80 x1862022294261632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768023 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767998.132101:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767998.132102:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775767998.132103:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767998.132105:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:0.0:1775767998.132107:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775767998.132108:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299922982 00000100:00000001:0.0:1775767998.132110:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299924285 00000100:00000001:0.0:1775767998.132111:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299922982 00000100:00000001:0.0:1775767998.132114:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299924285 00000100:00000001:0.0:1775767998.132115:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775767998.132117:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767998.132118:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775767998.132120:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767998/real 1775767998] req@ffff9942099aad80 x1862022294261632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775767998.132125:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775767998.132125:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775767998.132126:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775767998.132128:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775767998.132129:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775767998.132131:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775767998.132134:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099aad80 x1862022294261632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775767998.132143:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099aad80 x1862022294261632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767998.132150:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775767998.132151:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767998.132170:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767998.132172:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775767998.132177:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099aad80 x1862022294261632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767998.132184:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775767998.132188:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099aad80 x1862022294261632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768023 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767998.132193:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775767998.132195:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767998.132196:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775767998.132197:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767998.132199:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775767998.132201:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775767998.132203:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767998.132205:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775767998.132207:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775767998.132209:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099aad80 x1862022294261632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768023 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767998.132214:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099aad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294261632:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775767998.132216:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767998.132217:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775767998.132219:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099aad80 x1862022294261632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768023 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775767998.132223:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775767998.132224:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775767998.132225:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220028400. 02000000:00000001:0.0:1775767998.132227:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775767998.132228:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775767998.132229:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:0.0:1775767998.132230:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775767998.132231:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b3400. 02000000:00000001:0.0:1775767998.132233:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775767998.132234:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775767998.132235:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099aad80. 00000100:00000001:0.0:1775767998.132236:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775767998.132237:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767998.132238:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767998.132241:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767998.132242:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767998.132243:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775767998.132244:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767998.193249:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767998.193254:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767998.193261:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767998.193263:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767998.193265:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767998.193266:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767998.193268:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767998.193269:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767998.193271:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767998.193272:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767998.193296:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767998.193298:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767998.193300:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767998.257264:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767998.257270:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767998.257272:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767998.257276:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767998.257279:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767998.257282:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767998.257284:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767998.257287:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767998.257289:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767999.153224:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775767999.153232:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767999.153234:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767999.153239:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:2.0:1775767999.153240:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775767999.153245:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775767999.153246:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775767999.153248:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775767999.153251:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775767999.153253:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775767999.153255:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767999.153259:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775767999.153260:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767999.153264:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775767999.153265:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775767999.153270:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775767999.153274:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775767999.153277:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775767999.153290:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767999.153294:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767999.153295:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767999.153298:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775767999.153301:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767999.153303:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767999.153305:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767999.153306:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767999.153308:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767999.217266:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767999.217271:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767999.217276:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767999.217278:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767999.217281:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767999.217282:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767999.217284:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767999.217285:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775767999.217287:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775767999.217288:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775767999.217335:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775767999.217337:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775767999.217348:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775767999.281225:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775767999.281230:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775767999.281231:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775767999.281236:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775767999.281239:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775767999.281242:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767999.281244:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775767999.281246:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775767999.281248:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775767999.473244:0:14953:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4299924285, at 4299924288 00000100:00000040:1.0:1775767999.473244:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299924285, at 4299924288 00000100:00000010:0.0:1775767999.473259:0:14953:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3d80. 00000100:00000010:1.0:1775767999.473262:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3ea0. 00000100:00000001:2.0:1775768000.177224:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768000.177231:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768000.177232:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768000.177233:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768000.177235:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768000.177236:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768000.177239:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768000.177239:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:0.0:1775768000.177240:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768000.177244:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768000.177246:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768000.177246:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:0.0:1775768000.177247:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768000.177251:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768000.177252:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768000.177252:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768000.177254:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768000.177256:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768000.177257:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768000.177257:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768000.177263:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768000.177264:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768000.177267:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768000.177270:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768000.177270:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768000.177273:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768000.177275:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768000.241257:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768000.241263:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768000.241269:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768000.241270:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768000.241274:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768000.241276:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768000.241279:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768000.241280:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768000.241283:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768000.241285:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768000.241314:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768000.241317:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768000.241320:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768000.305284:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768000.305291:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768000.305293:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768000.305298:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768000.305302:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768000.305306:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768000.305309:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768000.305311:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768000.305314:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768001.201221:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768001.201229:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768001.201231:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768001.201238:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768001.201243:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768001.201243:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768001.201249:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768001.201249:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768001.201252:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768001.201254:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768001.201257:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768001.201258:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768001.201261:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768001.201263:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768001.201267:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768001.201269:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768001.201272:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768001.201275:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768001.201457:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768001.201459:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768001.201460:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768001.201463:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768001.201465:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768001.201466:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768001.201467:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768001.201469:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768001.201470:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768001.265277:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768001.265284:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768001.265287:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768001.265290:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768001.265295:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768001.265298:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768001.265300:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768001.265302:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768001.265306:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768001.265308:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768001.265312:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768001.265313:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768001.265317:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768001.265318:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768001.265321:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768001.265322:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768001.265336:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768001.265338:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768001.265339:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768001.329230:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768001.329236:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768001.329238:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768001.329243:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768001.329246:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768001.329249:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768001.329251:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768001.329253:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768001.329255:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768002.225905:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768002.225918:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768002.225919:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768002.225921:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768002.225927:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:1.0:1775768002.225931:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775768002.225931:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768002.225938:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:2.0:1775768002.225941:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768002.225950:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768002.225955:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768002.225960:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768002.225961:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768002.225967:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768002.225968:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768002.225978:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768002.225983:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768002.225987:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768002.226206:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768002.226210:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768002.226211:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768002.226214:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768002.226216:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768002.226218:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768002.226219:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768002.226221:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768002.226223:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768002.289276:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768002.289285:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768002.289293:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768002.289294:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768002.289300:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768002.289302:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768002.289306:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768002.289308:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768002.289312:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768002.289313:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768002.289356:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768002.289359:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768002.289363:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768002.353297:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768002.353316:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768002.353319:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768002.353326:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768002.353331:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768002.353336:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768002.353339:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768002.353343:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768002.353347:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768003.249315:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768003.249319:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768003.249324:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768003.249325:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768003.249330:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768003.249331:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768003.249342:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:1.0:1775768003.249343:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768003.249352:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768003.249352:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768003.249357:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768003.249358:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768003.249360:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768003.249363:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768003.249366:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768003.249371:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768003.249375:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768003.249379:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768003.249398:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768003.249405:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768003.249408:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768003.249412:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768003.249419:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768003.249425:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768003.249429:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:2.0:1775768003.249430:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775768003.249431:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768003.249438:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768003.249439:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775768003.249446:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775768003.249450:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775768003.249456:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775768003.249465:0:13471:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5255 00000400:00000200:2.0:1775768003.249475:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768003.249484:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775768003.249488:0:13471:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775768003.249491:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768003.249495:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768003.249498:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768003.249500:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775768003.249505:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775768003.249507:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:2.0:1775768003.249510:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775768003.249513:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:2.0:1775768003.249515:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:2.0:1775768003.249518:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:2.0:1775768003.249521:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768003.249523:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768003.249524:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768003.249527:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775768003.249530:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:2.0:1775768003.249534:0:13471:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775768003.249537:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768003.249541:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775768003.249545:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768003.249549:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768003.249551:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775768003.249554:0:13471:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775768003.249563:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfce00. 00000020:00000040:2.0:1775768003.249567:0:13471:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:2.0:1775768003.249570:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775768003.249572:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768003.249575:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775768003.249579:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775768003.249583:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775768003.249592:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1e400. 00000100:00000001:2.0:1775768003.249598:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768003.249607:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfce00 x1862022294261760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775768003.249616:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775768003.249620:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768003.249625:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e5a7debe] to pc [ptlrpcd_rcv+-1] req@ffff994207dfce00 x1862022294261760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768003.249656:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768003.249660:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768003.249664:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775768003.249667:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775768003.249669:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775768003.249673:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:0.0:1775768003.249677:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:2.0:1775768003.249678:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5255 00000100:00000001:0.0:1775768003.249684:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768003.249688:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000400:00000200:2.0:1775768003.249688:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768003.249698:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000040:0.0:1775768003.249705:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfce00 x1862022294261760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768003.249705:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775768003.249719:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768003.249729:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1775768003.249731:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768003.249734:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768003.249734:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768003.249743:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 02000000:00000001:0.0:1775768003.249746:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:2.0:1775768003.249746:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775768003.249750:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 02000000:00000001:0.0:1775768003.249752:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1775768003.249754:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000001:0.0:1775768003.249760:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775768003.249761:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:2.0:1775768003.249767:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00100000:0.0:1775768003.249770:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294261760:192.168.202.156@tcp:250:kworker.0 00000020:00000001:2.0:1775768003.249775:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000001:0.0:1775768003.249779:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768003.249785:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768003.249787:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768003.249791:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:2.0:1775768003.249792:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768003.249796:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000010:0.0:1775768003.249799:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002bc00. 00000100:00000040:2.0:1775768003.249801:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000001:0.0:1775768003.249807:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768003.249807:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775768003.249816:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000400:00000200:0.0:1775768003.249823:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00080000:2.0:1775768003.249823:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000400:00000010:0.0:1775768003.249830:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000100:00000001:2.0:1775768003.249832:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768003.249836:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:0.0:1775768003.249841:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 02000000:00000001:2.0:1775768003.249842:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000200:0.0:1775768003.249853:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294261760, portal 25 02000000:00000001:2.0:1775768003.249854:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768003.249860:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768003.249861:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775768003.249876:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611840000 : -112966097711616 : ffff994207dfce00) 00000008:00100000:2.0:1775768003.249880:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775768003.249884:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768003.249887:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768003.249889:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfce00 x1862022294261760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768028 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1775768003.249892:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dff480. 00000020:00000040:2.0:1775768003.249898:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:2.0:1775768003.249908:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775768003.249911:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768003.249921:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1775768003.249922:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:2.0:1775768003.249923:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775768003.249926:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768003.249932:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 02000000:00000010:2.0:1775768003.249935:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1f800. 00000400:00000010:0.0:1775768003.249937:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000001:2.0:1775768003.249940:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1775768003.249942:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294261760, offset 0 00000100:00100000:2.0:1775768003.249947:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dff480 x1862022294261888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1775768003.249958:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00000001:2.0:1775768003.249967:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775768003.249969:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768003.249973:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093a15639] to pc [ptlrpcd_rcv+-1] req@ffff994207dff480 x1862022294261888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768003.249980:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768003.249986:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00080000:2.0:1775768003.249989:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768003.249993:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775768003.249997:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:0.0:1775768003.249998:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00080000:2.0:1775768003.250000:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:0.0:1775768003.250006:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1775768003.250006:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775768003.250012:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5255 00000400:00000200:0.0:1775768003.250014:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768003.250025:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768003.250030:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768003.250034:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768003.250040:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225218:225218:256:4294967295] 192.168.202.156@tcp LPNI seq info [225214:225214:8:4294967295] 00000400:00000200:2.0:1775768003.250044:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775768003.250048:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000400:00000200:0.0:1775768003.250052:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000040:2.0:1775768003.250058:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768003.250061:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768003.250063:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768003.250065:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775768003.250072:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775768003.250077:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000400:00000200:0.0:1775768003.250082:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000020:00000001:2.0:1775768003.250082:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775768003.250086:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:2.0:1775768003.250093:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:2.0:1775768003.250099:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:2.0:1775768003.250107:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000800:00000200:0.0:1775768003.250109:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00000001:2.0:1775768003.250112:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768003.250116:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000800:00000010:0.0:1775768003.250118:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000100:00000001:2.0:1775768003.250120:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775768003.250125:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000800:00000200:0.0:1775768003.250133:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00080000:2.0:1775768003.250134:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000800:00000200:0.0:1775768003.250141:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000100:00000001:2.0:1775768003.250142:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768003.250147:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775768003.250152:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768003.250188:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768003.250192:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000800:00000200:0.1:1775768003.250192:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000008:00100000:2.0:1775768003.250197:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000200:0.1:1775768003.250199:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000008:00000001:2.0:1775768003.250202:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768003.250208:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775768003.250216:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfed80. 00000020:00000040:2.0:1775768003.250220:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:2.0:1775768003.250224:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775768003.250227:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1775768003.250231:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000100:00000001:2.0:1775768003.250232:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000800:00000010:0.0:1775768003.250235:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 02000000:00000001:2.0:1775768003.250236:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775768003.250244:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768003.250246:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000010:2.0:1775768003.250250:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1d000. 00000100:00000001:2.0:1775768003.250256:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768003.250257:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768003.250266:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000100:00100000:2.0:1775768003.250266:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfed80 x1862022294262016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:0.0:1775768003.250272:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775768003.250281:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1775768003.250282:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775768003.250289:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768003.250294:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb9b9eae] to pc [ptlrpcd_rcv+-1] req@ffff994207dfed80 x1862022294262016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768003.250295:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfce00 x1862022294261760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768028 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768003.250312:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768003.250314:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768003.250329:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1775768003.250336:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775768003.250340:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775768003.250343:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:0.0:1775768003.250346:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00080000:2.0:1775768003.250347:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5255 00000400:00000200:2.0:1775768003.250356:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000040:0.0:1775768003.250357:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfce00 x1862022294261760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768028 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1775768003.250362:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775768003.250370:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775768003.250380:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768003.250388:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768003.250395:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768003.250400:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768003.250401:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:0.0:1775768003.250410:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:2.0:1775768003.250410:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775768003.250418:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:2.0:1775768003.250421:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775768003.250423:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000100:00000001:0.0:1775768003.250424:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768003.250425:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:2.0:1775768003.250427:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:2.0:1775768003.250436:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768003.250437:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768003.250441:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000200:0.0:1775768003.250446:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768003.250447:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775768003.250452:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00000001:0.0:1775768003.250454:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299929406 00000100:00080000:2.0:1775768003.250457:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775768003.250461:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768003.250462:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775768003.250465:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768003.250467:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768003.250469:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775768003.250472:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:2.0:1775768003.250474:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768003.250476:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775768003.250480:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfe300. 00000020:00000040:2.0:1775768003.250483:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:2.0:1775768003.250485:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775768003.250487:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768003.250489:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775768003.250491:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775768003.250494:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775768003.250498:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1f400. 00000100:00000001:2.0:1775768003.250504:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768003.250513:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfe300 x1862022294262144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775768003.250523:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775768003.250526:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768003.250533:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c661fef] to pc [ptlrpcd_rcv+-1] req@ffff994207dfe300 x1862022294262144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1775768003.250534:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942078a3d80. 00000100:00000001:0.0:1775768003.250546:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1775768003.250550:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768003.250555:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299929406 00000100:00080000:2.0:1775768003.250555:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000010:0.0:1775768003.250565:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942078a3240. 00000100:00080000:2.0:1775768003.250567:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1775768003.250571:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1775768003.250572:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00080000:2.0:1775768003.250573:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775768003.250579:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:0.0:1775768003.250581:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1775768003.250584:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5255 00000100:00000001:0.0:1775768003.250588:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768003.250596:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768003.250597:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000400:00000200:2.0:1775768003.250602:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00100000:0.0:1775768003.250607:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768003/real 1775768003] req@ffff994207dfce00 x1862022294261760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768003.250608:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775768003.250622:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768003.250630:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768003.250641:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768003.250653:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000001:0.0:1775768003.250655:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:2.0:1775768003.250657:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000400:00000001:0.0:1775768003.250658:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000040:2.0:1775768003.250662:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000400:00000010:0.0:1775768003.250663:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000020:00000001:2.0:1775768003.250667:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:0.0:1775768003.250669:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000020:00000040:2.0:1775768003.250670:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000400:00000010:0.0:1775768003.250674:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000020:00000001:2.0:1775768003.250680:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000001:0.0:1775768003.250681:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:2.0:1775768003.250683:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000200:0.0:1775768003.250695:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfce00 x1862022294261760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775768003.250695:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768003.250705:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768003.250710:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768003.250719:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775768003.250725:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00000200:0.0:1775768003.250728:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfce00 x1862022294261760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768003.250732:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775768003.250742:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768003.250751:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1775768003.250758:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:2.0:1775768003.250758:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768003.250762:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1775768003.250764:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768003.250767:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768003.250768:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775768003.250770:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:2.0:1775768003.250772:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000100:00100000:0.0:1775768003.250778:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfce00 x1862022294261760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:2.0:1775768003.250779:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768003.250788:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775768003.250800:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfc700. 00000100:00000001:0.0:1775768003.250802:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:2.0:1775768003.250809:0:13471:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00000100:00000040:0.0:1775768003.250813:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfce00 x1862022294261760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768028 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1775768003.250818:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775768003.250822:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768003.250830:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775768003.250832:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775768003.250836:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768003.250837:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768003.250844:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:2.0:1775768003.250844:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1e800. 00000100:00000001:0.0:1775768003.250852:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768003.250853:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768003.250855:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768003.250859:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00100000:2.0:1775768003.250861:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfc700 x1862022294262272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768003.250874:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768003.250887:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768003.250901:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768003.250904:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768003.250905:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1775768003.250909:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768003.250915:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfce00 x1862022294261760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768028 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775768003.250920:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a176dd1] to pc [ptlrpcd_rcv+-1] req@ffff994207dfc700 x1862022294262272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768003.250946:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294261760:192.168.202.156@tcp:250:kworker.0 00000100:00000001:0.0:1775768003.250959:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768003.250961:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768003.250964:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768003.250969:0:13471:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5265) 00000100:00000040:0.0:1775768003.250970:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfce00 x1862022294261760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768028 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768003.250981:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768003.250984:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768003.250987:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002bc00. 02000000:00000001:0.0:1775768003.250993:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768003.250997:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768003.250999:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:0.0:1775768003.251002:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768003.251005:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1e400. 02000000:00000001:0.0:1775768003.251009:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768003.251011:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768003.251014:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfce00. 00000100:00000001:0.0:1775768003.251017:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768003.251018:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768003.251021:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768003.251028:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768003.251029:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768003.251032:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775768003.251034:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768003.251035:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768003.251039:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dff480 x1862022294261888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768003.251047:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768003.251048:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768003.251051:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768003.251053:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768003.251056:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768003.251058:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294261888:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768003.251062:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768003.251064:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768003.251066:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768003.251067:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768003.251071:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002bc00. 02000000:00000001:0.0:1775768003.251073:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768003.251077:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768003.251080:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775768003.251083:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768003.251086:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294261888, portal 10 00000100:00000001:0.0:1775768003.251088:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768003.251089:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611849856 : -112966097701760 : ffff994207dff480) 00000100:00000040:0.0:1775768003.251096:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dff480 x1862022294261888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768028 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768003.251102:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768003.251104:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768003.251106:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775768003.251108:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294261888, offset 0 00000400:00000200:0.0:1775768003.251112:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768003.251116:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768003.251119:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768003.251122:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768003.251125:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768003.251127:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768003.251130:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768003.251133:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225219:225219:256:4294967295] 192.168.202.156@tcp LPNI seq info [225215:225215:8:4294967295] 00000400:00000200:0.0:1775768003.251138:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768003.251143:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768003.251148:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768003.251151:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775768003.251183:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768003.251187:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1775768003.251193:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768003.251195:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000100:0.0:1775768003.251200:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768003.251203:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775768003.251206:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768003.251210:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768003.251213:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775768003.251214:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775768003.251216:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768003.251220:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dff480 x1862022294261888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768028 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768003.251228:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768003.251230:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dff480 x1862022294261888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768028 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768003.251237:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768003.251239:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768003.251241:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768003.251245:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:0.0:1775768003.251248:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768003.251250:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299928084 00000100:00000001:0.0:1775768003.251253:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299929406 00000100:00000001:0.0:1775768003.251256:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299928084 00000100:00000001:0.0:1775768003.251258:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299929406 00000100:00000001:0.0:1775768003.251260:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768003.251262:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768003.251264:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768003.251267:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768003/real 1775768003] req@ffff994207dff480 x1862022294261888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768003.251273:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768003.251275:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768003.251277:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775768003.251279:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768003.251280:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768003.251282:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768003.251285:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dff480 x1862022294261888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768003.251293:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dff480 x1862022294261888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768003.251300:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768003.251301:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768003.251303:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768003.251304:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768003.251307:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dff480 x1862022294261888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768003.251314:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768003.251317:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dff480 x1862022294261888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768028 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768003.251324:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768003.251326:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768003.251327:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768003.251328:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768003.251330:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768003.251332:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768003.251334:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768003.251338:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768003.251340:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768003.251343:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dff480 x1862022294261888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768028 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768003.251351:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294261888:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768003.251355:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768003.251359:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfed80 x1862022294262016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768003.251365:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768003.251366:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768003.251368:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768003.251370:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768003.251372:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768003.251375:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294262016:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768003.251379:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768003.251381:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768003.251382:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768003.251384:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768003.251387:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220029400. 02000000:00000001:0.0:1775768003.251389:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768003.251392:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768003.251394:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775768003.251397:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768003.251400:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294262016, portal 10 00000100:00000001:0.0:1775768003.251401:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768003.251403:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611848064 : -112966097703552 : ffff994207dfed80) 00000100:00000040:0.0:1775768003.251408:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfed80 x1862022294262016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768028 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768003.251415:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768003.251417:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768003.251419:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775768003.251421:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294262016, offset 0 00000400:00000200:0.0:1775768003.251424:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768003.251428:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768003.251431:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768003.251433:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768003.251436:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768003.251438:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768003.251441:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768003.251444:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225220:225220:256:4294967295] 192.168.202.156@tcp LPNI seq info [225216:225216:8:4294967295] 00000400:00000200:0.0:1775768003.251449:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768003.251455:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768003.251459:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768003.251462:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775768003.251466:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768003.251468:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1775768003.251472:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768003.251475:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000100:0.0:1775768003.251478:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768003.251480:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775768003.251482:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768003.251486:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768003.251488:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775768003.251490:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775768003.251492:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768003.251495:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfed80 x1862022294262016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768028 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768003.251502:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768003.251505:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfed80 x1862022294262016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768028 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768003.251512:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768003.251513:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768003.251516:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768003.251519:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:0.0:1775768003.251522:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768003.251523:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299928076 00000100:00000001:0.0:1775768003.251526:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299929406 00000100:00000001:0.0:1775768003.251528:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299928076 00000100:00000001:0.0:1775768003.251531:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299929406 00000100:00000001:0.0:1775768003.251533:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768003.251535:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768003.251537:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768003.251540:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768003/real 1775768003] req@ffff994207dfed80 x1862022294262016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768003.251547:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768003.251548:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768003.251550:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775768003.251552:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768003.251553:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768003.251555:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768003.251558:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfed80 x1862022294262016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768003.251566:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfed80 x1862022294262016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768003.251572:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768003.251574:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768003.251575:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768003.251576:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768003.251580:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfed80 x1862022294262016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768003.251587:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768003.251590:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfed80 x1862022294262016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768028 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768003.251596:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768003.251598:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768003.251599:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768003.251600:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768003.251602:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768003.251605:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768003.251608:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768003.251611:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768003.251613:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768003.251617:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfed80 x1862022294262016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768028 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768003.251624:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294262016:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768003.251628:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768003.251632:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfe300 x1862022294262144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768003.251638:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768003.251640:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768003.251642:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768003.251644:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768003.251646:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768003.251648:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294262144:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768003.251651:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768003.251653:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768003.251655:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768003.251656:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768003.251659:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002b800. 02000000:00000001:0.0:1775768003.251661:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768003.251665:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768003.251667:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775768003.251670:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768003.251672:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294262144, portal 4 00000100:00000001:0.0:1775768003.251674:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768003.251675:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611845376 : -112966097706240 : ffff994207dfe300) 00000100:00000040:0.0:1775768003.251680:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfe300 x1862022294262144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768028 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768003.251687:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768003.251689:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768003.251691:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775768003.251693:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294262144, offset 0 00000400:00000200:0.0:1775768003.251696:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768003.251699:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768003.251703:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768003.251705:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768003.251707:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768003.251709:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768003.251712:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768003.251715:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225221:225221:256:4294967295] 192.168.202.156@tcp LPNI seq info [225217:225217:8:4294967295] 00000400:00000200:0.0:1775768003.251719:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768003.251724:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768003.251729:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768003.251731:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775768003.251735:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768003.251738:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1775768003.251742:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768003.251744:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000100:0.0:1775768003.251748:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768003.251750:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775768003.251752:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768003.251756:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768003.251758:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775768003.251760:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775768003.251762:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768003.251765:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfe300 x1862022294262144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768028 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768003.251772:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768003.251775:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfe300 x1862022294262144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768028 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768003.251787:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768003.251789:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768003.251791:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768003.251794:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:0.0:1775768003.251797:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768003.251799:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299928099 00000100:00000001:0.0:1775768003.251801:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299929406 00000100:00000001:0.0:1775768003.251803:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299928099 00000100:00000001:0.0:1775768003.251805:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299929406 00000100:00000001:0.0:1775768003.251807:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768003.251809:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768003.251811:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768003.251814:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768003/real 1775768003] req@ffff994207dfe300 x1862022294262144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768003.251822:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768003.251823:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768003.251824:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775768003.251826:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768003.251828:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768003.251830:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768003.251833:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfe300 x1862022294262144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768003.251841:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfe300 x1862022294262144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768003.251847:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768003.251848:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768003.251850:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768003.251851:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768003.251855:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfe300 x1862022294262144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768003.251874:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768003.251878:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfe300 x1862022294262144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768028 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768003.251885:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768003.251887:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768003.251889:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768003.251890:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768003.251891:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768003.251895:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768003.251898:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768003.251901:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768003.251903:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768003.251907:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfe300 x1862022294262144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768028 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768003.251914:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294262144:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768003.251918:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768003.251922:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfc700 x1862022294262272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768003.251928:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768003.251930:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768003.251932:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768003.251933:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768003.251935:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768003.251938:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294262272:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768003.251942:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768003.251943:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768003.251945:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768003.251947:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768003.251950:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002a800. 02000000:00000001:0.0:1775768003.251951:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768003.251954:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768003.251957:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775768003.251959:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768003.251962:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294262272, portal 4 00000100:00000001:0.0:1775768003.251963:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768003.251965:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611838208 : -112966097713408 : ffff994207dfc700) 00000100:00000040:0.0:1775768003.251970:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfc700 x1862022294262272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768028 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768003.251976:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768003.251978:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768003.251980:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775768003.251982:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294262272, offset 0 00000400:00000200:0.0:1775768003.251985:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768003.251989:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768003.251992:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768003.251994:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768003.251997:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768003.251999:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768003.252001:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768003.252005:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225222:225222:256:4294967295] 192.168.202.156@tcp LPNI seq info [225218:225218:8:4294967295] 00000400:00000200:0.0:1775768003.252008:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768003.252014:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768003.252018:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768003.252021:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775768003.252024:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768003.252027:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1775768003.252031:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768003.252033:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000100:0.0:1775768003.252037:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768003.252039:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775768003.252042:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768003.252045:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768003.252048:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775768003.252049:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775768003.252051:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768003.252054:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfc700 x1862022294262272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768028 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768003.252061:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768003.252064:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfc700 x1862022294262272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768028 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768003.252071:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768003.252073:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768003.252075:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768003.252078:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:0.0:1775768003.252081:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768003.252082:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299928102 00000100:00000001:0.0:1775768003.252085:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299929406 00000100:00000001:0.0:1775768003.252087:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299928102 00000100:00000001:0.0:1775768003.252089:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299929406 00000100:00000001:0.0:1775768003.252091:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768003.252093:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768003.252094:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768003.252097:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768003/real 1775768003] req@ffff994207dfc700 x1862022294262272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768003.252103:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768003.252105:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768003.252106:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775768003.252108:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768003.252110:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768003.252112:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768003.252115:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfc700 x1862022294262272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768003.252123:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfc700 x1862022294262272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768003.252129:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768003.252131:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768003.252132:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768003.252133:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768003.252136:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfc700 x1862022294262272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768003.252142:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768003.252146:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfc700 x1862022294262272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768028 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768003.252152:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768003.252176:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768003.252178:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768003.252180:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768003.252182:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768003.252186:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768003.252189:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768003.252192:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768003.252194:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768003.252198:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfc700 x1862022294262272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768028 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768003.252206:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294262272:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768003.252209:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768003.252211:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768003.252214:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dff480 x1862022294261888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768028 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768003.252221:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768003.252222:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768003.252224:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002bc00. 02000000:00000001:0.0:1775768003.252226:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768003.252228:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768003.252229:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:0.0:1775768003.252231:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768003.252233:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1f800. 02000000:00000001:0.0:1775768003.252236:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768003.252237:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768003.252239:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dff480. 00000100:00000001:0.0:1775768003.252241:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768003.252243:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768003.252244:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768003.252247:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfed80 x1862022294262016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768028 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768003.252254:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768003.252255:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768003.252256:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220029400. 02000000:00000001:0.0:1775768003.252258:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768003.252259:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768003.252260:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:0.0:1775768003.252262:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768003.252263:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1d000. 02000000:00000001:0.0:1775768003.252267:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768003.252268:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768003.252269:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfed80. 00000100:00000001:0.0:1775768003.252272:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768003.252273:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768003.252275:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768003.252277:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfe300 x1862022294262144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768028 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768003.252283:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768003.252284:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768003.252286:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002b800. 02000000:00000001:0.0:1775768003.252288:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768003.252289:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768003.252290:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:0.0:1775768003.252292:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768003.252293:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1f400. 02000000:00000001:0.0:1775768003.252295:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768003.252296:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768003.252297:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfe300. 00000100:00000001:0.0:1775768003.252299:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768003.252300:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768003.252301:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768003.252304:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfc700 x1862022294262272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768028 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768003.252309:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768003.252310:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768003.252311:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002a800. 02000000:00000001:0.0:1775768003.252314:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768003.252315:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768003.252316:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:0.0:1775768003.252318:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768003.252319:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1e800. 02000000:00000001:0.0:1775768003.252321:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768003.252322:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768003.252323:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfc700. 00000100:00000001:0.0:1775768003.252325:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768003.252326:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768003.252327:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768003.252331:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768003.252332:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768003.252334:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775768003.252335:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768003.313296:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768003.313304:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768003.313310:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768003.313311:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768003.313316:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768003.313318:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768003.313322:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768003.313324:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768003.313327:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768003.313328:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768003.313349:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768003.313352:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768003.313355:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768003.377241:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768003.377249:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768003.377252:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768003.377258:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768003.377263:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768003.377267:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768003.377270:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768003.377273:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768003.377276:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768004.273245:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768004.273253:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768004.273255:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768004.273260:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:2.0:1775768004.273261:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768004.273265:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768004.273266:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768004.273266:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768004.273269:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768004.273270:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768004.273272:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768004.273273:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768004.273274:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768004.273275:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768004.273278:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768004.273280:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768004.273284:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768004.273285:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768004.273286:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00100000:1.0:1775768004.273288:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775768004.273290:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768004.273292:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768004.273294:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768004.273296:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768004.273302:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768004.273304:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768004.273307:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768004.337296:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768004.337304:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768004.337312:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768004.337314:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768004.337318:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768004.337319:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768004.337323:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768004.337324:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768004.337328:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768004.337329:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768004.337354:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768004.337357:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768004.337360:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768004.401227:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768004.401232:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768004.401234:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768004.401238:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768004.401240:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768004.401244:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768004.401246:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768004.401248:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768004.401250:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768004.593241:0:14933:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299929406, at 4299929408 00000100:00000010:1.0:1775768004.593259:0:14933:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3240. 00000100:00000040:0.0:1775768004.593277:0:14953:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4299929406, at 4299929408 00000100:00000010:0.0:1775768004.593289:0:14953:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3d80. 00000100:00000001:1.0:1775768005.297253:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768005.297253:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768005.297259:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768005.297260:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768005.297262:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768005.297267:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768005.297267:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768005.297273:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768005.297273:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768005.297274:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768005.297276:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768005.297278:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768005.297279:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768005.297280:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768005.297285:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775768005.297285:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768005.297286:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768005.297289:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768005.297291:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768005.297291:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768005.297296:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768005.297296:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768005.297301:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768005.297304:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768005.297307:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768005.297311:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768005.297314:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768005.361292:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768005.361299:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768005.361306:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768005.361307:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768005.361312:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768005.361313:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768005.361317:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768005.361318:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768005.361322:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768005.361323:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768005.361376:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768005.361380:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768005.361384:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768005.425325:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768005.425341:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768005.425343:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768005.425349:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768005.425353:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768005.425357:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768005.425360:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768005.425362:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768005.425365:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768006.321284:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768006.321288:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768006.321299:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768006.321299:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768006.321307:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768006.321308:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768006.321308:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768006.321315:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768006.321318:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768006.321320:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775768006.321320:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:0.0:1775768006.321321:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775768006.321335:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768006.321336:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768006.321336:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768006.321345:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768006.321346:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768006.321346:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768006.321359:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768006.321362:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768006.321366:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768006.321367:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768006.321369:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768006.321371:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768006.321373:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768006.321377:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768006.321381:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768006.385281:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768006.385290:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768006.385298:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768006.385302:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768006.385305:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768006.385309:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768006.385313:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768006.385315:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768006.385318:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768006.385320:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768006.385325:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768006.385327:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768006.385328:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768006.385329:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768006.385330:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768006.385331:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768006.385332:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768006.385334:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768006.385336:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768006.385337:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768006.385338:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768006.385339:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768006.385340:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768006.385340:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768006.385341:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768006.385342:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768006.385345:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768006.385346:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768006.385346:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768006.385347:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768006.385348:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768006.385349:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768006.385349:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768006.385351:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768006.385403:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768006.385406:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768006.385409:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768006.449264:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768006.449273:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768006.449276:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768006.449283:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768006.449288:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768006.449293:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768006.449296:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768006.449298:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768006.449301:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768007.345243:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768007.345243:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768007.345250:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768007.345250:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768007.345252:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768007.345252:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768007.345259:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775768007.345259:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768007.345264:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768007.345264:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768007.345268:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768007.345268:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768007.345271:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768007.345274:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768007.345277:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768007.345277:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768007.345278:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768007.345281:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768007.345283:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768007.345284:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768007.345287:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768007.345292:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768007.345296:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768007.345299:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768007.345302:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768007.345304:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768007.345307:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768007.409273:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768007.409280:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768007.409287:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768007.409289:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768007.409293:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768007.409295:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768007.409298:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768007.409300:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768007.409303:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768007.409304:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768007.409324:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768007.409327:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768007.409330:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768007.473271:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768007.473278:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768007.473280:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768007.473285:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768007.473289:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768007.473293:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768007.473296:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768007.473299:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768007.473301:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768008.369267:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768008.369269:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768008.369274:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768008.369275:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768008.369278:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768008.369283:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768008.369283:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:0.0:1775768008.369289:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775768008.369289:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768008.369293:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768008.369294:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768008.369296:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768008.369298:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768008.369301:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768008.369304:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768008.369307:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768008.369310:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768008.369313:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768008.369873:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768008.369876:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768008.369877:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768008.369880:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775768008.369882:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768008.369883:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768008.369885:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768008.369886:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768008.369888:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768008.369913:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768008.369920:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775768008.369923:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775768008.369928:0:8:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768008.369932:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775768008.369937:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5260 00000400:00000200:1.0:1775768008.369945:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768008.369951:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775768008.369954:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775768008.369956:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768008.369959:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768008.369961:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768008.369962:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775768008.369965:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775768008.369966:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:1.0:1775768008.369968:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775768008.369969:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:1.0:1775768008.369971:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:1.0:1775768008.369972:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:1.0:1775768008.369974:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768008.369975:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768008.369976:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768008.369977:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775768008.369984:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:1.0:1775768008.369986:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775768008.369987:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768008.369989:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775768008.369992:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768008.369993:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768008.369995:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775768008.369996:0:8:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775768008.370004:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207888e00. 00000020:00000040:1.0:1775768008.370007:0:8:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:1.0:1775768008.370012:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768008.370013:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768008.370017:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775768008.370019:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775768008.370024:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775768008.370032:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543d400. 00000100:00000001:1.0:1775768008.370039:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768008.370046:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207888e00 x1862022294262400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775768008.370053:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775768008.370056:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768008.370059:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001812f966] to pc [ptlrpcd_rcv+-1] req@ffff994207888e00 x1862022294262400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768008.370079:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768008.370082:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768008.370084:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775768008.370086:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775768008.370087:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768008.370090:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775768008.370092:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5260 00000400:00000200:1.0:1775768008.370096:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768008.370100:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775768008.370102:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775768008.370106:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768008.370108:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768008.370110:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768008.370112:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768008.370113:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:0.0:1775768008.370117:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:1.0:1775768008.370117:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775768008.370118:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000100:00000001:0.0:1775768008.370120:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000001:1.0:1775768008.370121:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775768008.370124:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:1.0:1775768008.370128:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:0.0:1775768008.370129:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207888e00 x1862022294262400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1775768008.370132:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:1.0:1775768008.370140:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768008.370145:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1775768008.370148:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000040:1.0:1775768008.370149:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768008.370151:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768008.370153:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 02000000:00000001:0.0:1775768008.370176:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768008.370179:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768008.370182:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768008.370187:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207888e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294262400:192.168.202.156@tcp:250:kworker.0 00000100:00000001:0.0:1775768008.370192:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000040:1.0:1775768008.370196:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 02000000:00000001:0.0:1775768008.370198:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:1.0:1775768008.370200:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 02000000:00000001:0.0:1775768008.370202:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768008.370202:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768008.370206:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768008.370207:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1775768008.370211:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000010:0.0:1775768008.370214:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002b800. 02000000:00000001:1.0:1775768008.370214:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768008.370216:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:0.0:1775768008.370219:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:1.0:1775768008.370221:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775768008.370223:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768008.370224:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768008.370229:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000010:1.0:1775768008.370229:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207889880. 00000020:00000040:1.0:1775768008.370232:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000400:00000010:0.0:1775768008.370234:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 02000000:00000001:1.0:1775768008.370234:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768008.370235:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768008.370238:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775768008.370240:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775768008.370241:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775768008.370245:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 02000000:00000010:1.0:1775768008.370245:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543d000. 00000100:00000001:1.0:1775768008.370251:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1775768008.370252:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294262400, portal 25 00000100:00000001:0.0:1775768008.370257:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00100000:1.0:1775768008.370257:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207889880 x1862022294262528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768008.370260:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606121984 : -112966103429632 : ffff994207888e00) 00000100:00000040:0.0:1775768008.370270:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207888e00 x1862022294262400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768033 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768008.370271:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775768008.370275:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768008.370282:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f11856b8] to pc [ptlrpcd_rcv+-1] req@ffff994207889880 x1862022294262528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768008.370288:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768008.370296:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768008.370300:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000001:1.0:1775768008.370305:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1775768008.370306:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294262400, offset 0 00000100:00080000:1.0:1775768008.370310:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768008.370314:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000200:0.0:1775768008.370317:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00000001:1.0:1775768008.370318:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775768008.370320:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768008.370324:0:8:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:0.0:1775768008.370329:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00080000:1.0:1775768008.370329:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5260 00000400:00000200:0.0:1775768008.370338:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768008.370343:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768008.370347:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768008.370351:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768008.370353:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768008.370357:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225223:225223:256:4294967295] 192.168.202.156@tcp LPNI seq info [225219:225219:8:4294967295] 00000400:00000200:0.0:1775768008.370361:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768008.370372:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768008.370378:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768008.370383:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000800:00000010:0.0:1775768008.370385:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000400:00000200:1.0:1775768008.370388:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775768008.370391:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000800:00000200:0.0:1775768008.370395:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000040:1.0:1775768008.370398:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000800:00000200:0.0:1775768008.370401:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 23 secs later) 00000100:00000001:1.0:1775768008.370402:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768008.370406:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768008.370410:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000800:00000200:0.1:1775768008.370411:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000001:1.0:1775768008.370414:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000800:00000200:0.1:1775768008.370416:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 23 secs later) 00000020:00000040:1.0:1775768008.370417:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:1.0:1775768008.370419:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775768008.370422:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:1.0:1775768008.370424:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000800:00000100:0.0:1775768008.370426:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000020:00000001:1.0:1775768008.370427:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000800:00000010:0.0:1775768008.370430:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000100:00000040:1.0:1775768008.370431:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768008.370435:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768008.370436:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000200:0.0:1775768008.370438:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:1.0:1775768008.370438:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775768008.370439:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:1.0:1775768008.370443:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775768008.370445:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768008.370448:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 02000000:00000001:1.0:1775768008.370449:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775768008.370454:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1775768008.370456:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 02000000:00000001:1.0:1775768008.370457:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775768008.370460:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:1.0:1775768008.370460:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775768008.370464:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1775768008.370466:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000008:00000001:1.0:1775768008.370468:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1775768008.370472:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207888e00 x1862022294262400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768033 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768008.370472:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775768008.370477:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207888700. 00000100:00000001:0.0:1775768008.370485:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768008.370488:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207888e00 x1862022294262400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768033 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1775768008.370496:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000100:00000001:0.0:1775768008.370498:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768008.370500:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1775768008.370502:0:2399:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:1.0:1775768008.370503:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768008.370506:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1775768008.370508:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768008.370508:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775768008.370510:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768008.370515:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000010:1.0:1775768008.370516:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543fc00. 00000100:00000001:0.0:1775768008.370518:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299934526 00000100:00000001:1.0:1775768008.370522:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768008.370525:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942078a3d80. 00000100:00100000:1.0:1775768008.370529:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207888700 x1862022294262656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768008.370534:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775768008.370541:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768008.370542:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299934526 00000100:00000001:1.0:1775768008.370543:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768008.370548:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942078a3840. 00000100:00000040:1.0:1775768008.370549:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009f2ba301] to pc [ptlrpcd_rcv+-1] req@ffff994207888700 x1862022294262656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768008.370555:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775768008.370556:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768008.370559:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775768008.370561:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:1.0:1775768008.370563:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775768008.370566:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768008.370566:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775768008.370568:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768008.370570:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000001:1.0:1775768008.370571:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775768008.370574:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5260 00000400:00000200:1.0:1775768008.370578:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00100000:0.0:1775768008.370579:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768008/real 1775768008] req@ffff994207888e00 x1862022294262400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775768008.370582:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775768008.370585:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000400:00000001:0.0:1775768008.370594:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768008.370596:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000040:1.0:1775768008.370597:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000010:0.0:1775768008.370598:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000100:00000001:1.0:1775768008.370601:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1775768008.370602:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000100:00000040:1.0:1775768008.370605:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000010:0.0:1775768008.370606:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:1.0:1775768008.370610:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:0.0:1775768008.370612:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:1.0:1775768008.370614:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775768008.370617:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000100:00000200:0.0:1775768008.370618:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207888e00 x1862022294262400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1775768008.370620:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775768008.370624:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:1.0:1775768008.370629:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:1.0:1775768008.370633:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000200:0.0:1775768008.370637:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207888e00 x1862022294262400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775768008.370640:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768008.370645:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1775768008.370651:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000040:1.0:1775768008.370651:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768008.370653:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:0.0:1775768008.370654:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:1.0:1775768008.370656:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00000001:0.0:1775768008.370658:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:1.0:1775768008.370660:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768008.370662:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768008.370663:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768008.370669:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207888e00 x1862022294262400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1775768008.370672:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775768008.370675:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768008.370679:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768008.370683:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775768008.370686:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:1.0:1775768008.370686:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:1.0:1775768008.370690:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768008.370692:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768008.370693:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207888e00 x1862022294262400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768033 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1775768008.370695:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff99420788b100. 00000020:00000040:1.0:1775768008.370699:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:1.0:1775768008.370704:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768008.370706:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768008.370709:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1775768008.370710:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:1.0:1775768008.370710:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775768008.370713:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768008.370715:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:1.0:1775768008.370716:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543dc00. 00000100:00000001:0.0:1775768008.370718:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768008.370721:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768008.370722:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768008.370724:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768008.370728:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00100000:1.0:1775768008.370728:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff99420788b100 x1862022294262784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768008.370732:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768008.370741:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768008.370742:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775768008.370745:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768008.370747:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768008.370749:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e26cf977] to pc [ptlrpcd_rcv+-1] req@ffff99420788b100 x1862022294262784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775768008.370756:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207888e00 x1862022294262400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768033 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768008.370757:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768008.370761:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768008.370763:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775768008.370767:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775768008.370769:0:8:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768008.370774:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775768008.370778:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5260 00000100:00100000:0.0:1775768008.370783:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207888e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294262400:192.168.202.156@tcp:250:kworker.0 00000400:00000200:1.0:1775768008.370784:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768008.370788:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775768008.370792:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1775768008.370793:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768008.370795:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768008.370797:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1775768008.370797:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768008.370801:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768008.370803:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000040:0.0:1775768008.370804:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207888e00 x1862022294262400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768033 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1775768008.370806:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775768008.370808:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:1.0:1775768008.370810:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775768008.370825:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000100:00000001:0.0:1775768008.370826:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768008.370829:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000001:1.0:1775768008.370830:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 02000000:00000010:0.0:1775768008.370833:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002b800. 00000020:00000001:1.0:1775768008.370833:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 02000000:00000001:0.0:1775768008.370839:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768008.370842:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000040:1.0:1775768008.370844:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000040:0.0:1775768008.370845:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:0.0:1775768008.370850:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1775768008.370850:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000010:0.0:1775768008.370853:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543d400. 00000100:00000040:1.0:1775768008.370854:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000001:0.0:1775768008.370858:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:1.0:1775768008.370858:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 02000000:00000001:0.0:1775768008.370860:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000020:00000040:1.0:1775768008.370862:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00000010:0.0:1775768008.370863:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207888e00. 00000100:00080000:1.0:1775768008.370867:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768008.370869:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768008.370870:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768008.370871:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1775768008.370874:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1775768008.370875:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1775768008.370878:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768008.370881:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768008.370882:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775768008.370884:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000008:00100000:1.0:1775768008.370884:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000100:00000001:0.0:1775768008.370887:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1775768008.370888:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768008.370890:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768008.370891:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768008.370893:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768008.370895:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000010:1.0:1775768008.370896:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207889f80. 00000100:00000040:0.0:1775768008.370900:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207889880 x1862022294262528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1775768008.370901:0:8:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:1.0:1775768008.370906:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768008.370908:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768008.370912:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775768008.370914:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1775768008.370916:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 02000000:00000001:1.0:1775768008.370917:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768008.370919:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775768008.370921:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543e400. 02000000:00000001:0.0:1775768008.370922:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1775768008.370924:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768008.370925:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768008.370929:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768008.370930:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207889f80 x1862022294262912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768008.370935:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207889880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294262528:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768008.370938:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775768008.370942:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768008.370943:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768008.370947:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:1.0:1775768008.370947:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ed2274f4] to pc [ptlrpcd_rcv+-1] req@ffff994207889f80 x1862022294262912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1775768008.370950:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768008.370954:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1775768008.370958:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768008.370959:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002b800. 00000100:00000040:1.0:1775768008.370961:0:8:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5270) 02000000:00000001:0.0:1775768008.370962:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768008.370968:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768008.370971:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775768008.370974:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768008.370977:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294262528, portal 10 00000100:00000001:0.0:1775768008.370990:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768008.370993:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606124672 : -112966103426944 : ffff994207889880) 00000100:00000040:0.0:1775768008.370999:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207889880 x1862022294262528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768033 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768008.371005:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768008.371007:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768008.371010:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775768008.371012:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294262528, offset 0 00000400:00000200:0.0:1775768008.371016:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768008.371020:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768008.371024:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768008.371026:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768008.371029:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768008.371031:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768008.371033:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768008.371036:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225224:225224:256:4294967295] 192.168.202.156@tcp LPNI seq info [225220:225220:8:4294967295] 00000400:00000200:0.0:1775768008.371040:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768008.371046:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768008.371049:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768008.371053:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775768008.371056:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768008.371059:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 23 secs later) 00000800:00000200:0.1:1775768008.371063:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768008.371065:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 23 secs later) 00000800:00000100:0.0:1775768008.371068:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768008.371069:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775768008.371072:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768008.371075:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768008.371077:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775768008.371079:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775768008.371081:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768008.371084:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207889880 x1862022294262528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768033 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768008.371090:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768008.371091:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207889880 x1862022294262528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768033 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768008.371096:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768008.371098:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768008.371100:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768008.371103:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:0.0:1775768008.371106:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768008.371108:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299933203 00000100:00000001:0.0:1775768008.371110:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299934526 00000100:00000001:0.0:1775768008.371112:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299933203 00000100:00000001:0.0:1775768008.371114:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299934526 00000100:00000001:0.0:1775768008.371116:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768008.371117:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768008.371119:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768008.371122:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768008/real 1775768008] req@ffff994207889880 x1862022294262528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768008.371128:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768008.371129:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768008.371131:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775768008.371133:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768008.371135:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768008.371137:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768008.371139:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207889880 x1862022294262528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768008.371147:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207889880 x1862022294262528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768008.371152:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768008.371168:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768008.371171:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768008.371172:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768008.371176:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207889880 x1862022294262528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768008.371182:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768008.371186:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207889880 x1862022294262528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768033 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768008.371192:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768008.371194:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768008.371196:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768008.371197:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768008.371198:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768008.371201:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768008.371203:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768008.371205:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768008.371207:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768008.371210:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207889880 x1862022294262528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768033 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768008.371217:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207889880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294262528:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768008.371221:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768008.371224:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207888700 x1862022294262656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768008.371229:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768008.371230:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768008.371232:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768008.371233:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768008.371235:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768008.371237:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207888700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294262656:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768008.371240:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768008.371242:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768008.371243:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768008.371245:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768008.371248:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220029400. 02000000:00000001:0.0:1775768008.371249:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768008.371252:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768008.371254:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775768008.371257:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768008.371260:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294262656, portal 10 00000100:00000001:0.0:1775768008.371261:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768008.371263:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606120192 : -112966103431424 : ffff994207888700) 00000100:00000040:0.0:1775768008.371267:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207888700 x1862022294262656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768033 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768008.371272:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768008.371274:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768008.371275:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775768008.371277:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294262656, offset 0 00000400:00000200:0.0:1775768008.371280:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768008.371284:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768008.371286:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768008.371289:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768008.371291:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768008.371293:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768008.371295:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768008.371297:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225225:225225:256:4294967295] 192.168.202.156@tcp LPNI seq info [225221:225221:8:4294967295] 00000400:00000200:0.0:1775768008.371300:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768008.371306:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768008.371309:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768008.371312:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775768008.371315:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768008.371317:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 23 secs later) 00000800:00000200:0.1:1775768008.371320:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768008.371322:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 23 secs later) 00000800:00000100:0.0:1775768008.371325:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768008.371327:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775768008.371329:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768008.371332:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768008.371334:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775768008.371336:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775768008.371338:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768008.371340:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207888700 x1862022294262656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768033 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768008.371351:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768008.371354:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207888700 x1862022294262656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768033 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768008.371359:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768008.371360:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768008.371362:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768008.371364:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:0.0:1775768008.371366:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768008.371368:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299933196 00000100:00000001:0.0:1775768008.371370:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299934526 00000100:00000001:0.0:1775768008.371372:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299933196 00000100:00000001:0.0:1775768008.371373:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299934526 00000100:00000001:0.0:1775768008.371375:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768008.371376:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768008.371378:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768008.371381:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768008/real 1775768008] req@ffff994207888700 x1862022294262656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768008.371388:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768008.371389:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768008.371390:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775768008.371392:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768008.371393:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768008.371395:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768008.371397:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207888700 x1862022294262656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768008.371405:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207888700 x1862022294262656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768008.371410:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768008.371412:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768008.371414:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768008.371415:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768008.371418:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207888700 x1862022294262656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768008.371424:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768008.371427:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207888700 x1862022294262656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768033 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768008.371433:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768008.371435:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768008.371437:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768008.371438:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768008.371439:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768008.371442:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768008.371444:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768008.371446:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768008.371448:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768008.371452:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207888700 x1862022294262656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768033 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768008.371460:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207888700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294262656:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768008.371463:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768008.371465:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff99420788b100 x1862022294262784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768008.371471:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768008.371472:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768008.371473:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768008.371475:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768008.371476:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768008.371479:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff99420788b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294262784:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768008.371481:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768008.371483:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768008.371484:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768008.371486:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768008.371488:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002bc00. 02000000:00000001:0.0:1775768008.371490:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768008.371492:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768008.371494:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775768008.371497:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768008.371499:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294262784, portal 4 00000100:00000001:0.0:1775768008.371501:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768008.371502:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606130944 : -112966103420672 : ffff99420788b100) 00000100:00000040:0.0:1775768008.371506:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff99420788b100 x1862022294262784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768033 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768008.371511:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768008.371512:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768008.371514:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775768008.371516:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294262784, offset 0 00000400:00000200:0.0:1775768008.371519:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768008.371523:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768008.371525:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768008.371527:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768008.371529:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768008.371531:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768008.371533:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768008.371536:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225226:225226:256:4294967295] 192.168.202.156@tcp LPNI seq info [225222:225222:8:4294967295] 00000400:00000200:0.0:1775768008.371539:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768008.371544:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768008.371547:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768008.371550:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775768008.371553:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768008.371556:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 23 secs later) 00000800:00000200:0.1:1775768008.371559:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768008.371561:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 23 secs later) 00000800:00000100:0.0:1775768008.371564:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768008.371566:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775768008.371569:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768008.371571:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768008.371574:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775768008.371575:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775768008.371577:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768008.371580:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff99420788b100 x1862022294262784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768033 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768008.371587:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768008.371590:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff99420788b100 x1862022294262784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768033 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768008.371596:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768008.371597:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768008.371600:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768008.371602:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:0.0:1775768008.371605:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768008.371607:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299933219 00000100:00000001:0.0:1775768008.371609:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299934526 00000100:00000001:0.0:1775768008.371610:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299933219 00000100:00000001:0.0:1775768008.371612:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299934526 00000100:00000001:0.0:1775768008.371614:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768008.371616:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768008.371618:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768008.371621:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768008/real 1775768008] req@ffff99420788b100 x1862022294262784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768008.371627:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768008.371629:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768008.371630:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775768008.371632:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768008.371633:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768008.371635:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768008.371637:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff99420788b100 x1862022294262784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768008.371644:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff99420788b100 x1862022294262784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768008.371649:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768008.371650:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768008.371651:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768008.371652:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768008.371655:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff99420788b100 x1862022294262784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768008.371661:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768008.371664:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff99420788b100 x1862022294262784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768033 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768008.371669:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768008.371671:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768008.371672:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768008.371673:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768008.371675:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768008.371678:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768008.371679:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768008.371682:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768008.371684:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768008.371687:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff99420788b100 x1862022294262784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768033 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768008.371698:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff99420788b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294262784:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768008.371701:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768008.371702:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768008.371704:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207889880 x1862022294262528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768033 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768008.371710:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768008.371711:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768008.371713:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002b800. 02000000:00000001:0.0:1775768008.371715:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768008.371716:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768008.371717:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:0.0:1775768008.371719:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768008.371720:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543d000. 02000000:00000001:0.0:1775768008.371722:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768008.371723:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768008.371724:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207889880. 00000100:00000001:0.0:1775768008.371726:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768008.371727:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768008.371729:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768008.371731:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207888700 x1862022294262656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768033 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768008.371736:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768008.371737:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768008.371739:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220029400. 02000000:00000001:0.0:1775768008.371741:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768008.371742:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768008.371743:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:0.0:1775768008.371744:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768008.371745:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543fc00. 02000000:00000001:0.0:1775768008.371747:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768008.371748:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768008.371749:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207888700. 00000100:00000001:0.0:1775768008.371752:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768008.371753:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768008.371754:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768008.371757:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff99420788b100 x1862022294262784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768033 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768008.371763:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768008.371764:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768008.371765:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002bc00. 02000000:00000001:0.0:1775768008.371767:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768008.371768:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768008.371769:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:0.0:1775768008.371770:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768008.371771:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543dc00. 02000000:00000001:0.0:1775768008.371773:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768008.371774:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768008.371775:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff99420788b100. 00000100:00000001:0.0:1775768008.371777:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768008.371778:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768008.371780:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768008.371783:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768008.371784:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768008.371785:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775768008.371787:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768008.371788:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768008.371791:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207889f80 x1862022294262912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768008.371796:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768008.371797:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768008.371799:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768008.371800:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768008.371802:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768008.371805:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207889f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294262912:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768008.371807:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768008.371809:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768008.371810:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768008.371825:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768008.371828:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002bc00. 02000000:00000001:0.0:1775768008.371830:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768008.371833:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768008.371834:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775768008.371837:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768008.371839:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294262912, portal 4 00000100:00000001:0.0:1775768008.371841:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768008.371842:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606126464 : -112966103425152 : ffff994207889f80) 00000100:00000040:0.0:1775768008.371846:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207889f80 x1862022294262912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768033 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768008.371851:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768008.371853:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768008.371855:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775768008.371857:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294262912, offset 0 00000400:00000200:0.0:1775768008.371860:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768008.371863:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768008.371866:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768008.371869:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768008.371871:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768008.371873:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768008.371875:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768008.371877:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225227:225227:256:4294967295] 192.168.202.156@tcp LPNI seq info [225223:225223:8:4294967295] 00000400:00000200:0.0:1775768008.371880:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768008.371885:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768008.371889:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768008.371891:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775768008.371894:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768008.371897:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 23 secs later) 00000800:00000200:0.1:1775768008.371900:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768008.371902:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 23 secs later) 00000800:00000100:0.0:1775768008.371905:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768008.371906:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775768008.371909:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768008.371912:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768008.371914:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775768008.371915:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775768008.371917:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768008.371920:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207889f80 x1862022294262912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768033 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768008.371926:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768008.371928:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207889f80 x1862022294262912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768033 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768008.371933:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768008.371935:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768008.371937:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768008.371940:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:0.0:1775768008.371942:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768008.371944:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299933222 00000100:00000001:0.0:1775768008.371946:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299934526 00000100:00000001:0.0:1775768008.371947:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299933222 00000100:00000001:0.0:1775768008.371949:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299934526 00000100:00000001:0.0:1775768008.371951:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768008.371952:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768008.371954:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768008.371957:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768008/real 1775768008] req@ffff994207889f80 x1862022294262912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768008.371963:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768008.371964:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768008.371966:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775768008.371968:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768008.371969:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768008.371971:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768008.371973:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207889f80 x1862022294262912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768008.371981:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207889f80 x1862022294262912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768008.371987:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768008.371988:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768008.371989:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768008.371991:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768008.371993:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207889f80 x1862022294262912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768008.371999:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768008.372002:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207889f80 x1862022294262912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768033 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768008.372008:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768008.372011:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768008.372012:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768008.372013:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768008.372015:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768008.372018:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768008.372020:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768008.372023:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768008.372025:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768008.372027:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207889f80 x1862022294262912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768033 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768008.372034:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207889f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294262912:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768008.372036:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768008.372038:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768008.372040:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207889f80 x1862022294262912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768033 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768008.372046:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768008.372047:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768008.372049:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002bc00. 02000000:00000001:0.0:1775768008.372051:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768008.372052:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768008.372053:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:0.0:1775768008.372055:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768008.372056:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543e400. 02000000:00000001:0.0:1775768008.372059:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768008.372060:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768008.372061:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207889f80. 00000100:00000001:0.0:1775768008.372063:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768008.372064:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768008.372066:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768008.372069:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768008.372070:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768008.372072:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775768008.372073:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768008.433242:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768008.433248:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768008.433253:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768008.433255:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768008.433258:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768008.433259:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768008.433262:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768008.433264:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768008.433267:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768008.433268:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768008.433285:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768008.433287:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768008.433289:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768008.497219:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768008.497226:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768008.497228:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768008.497232:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768008.497236:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768008.497240:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768008.497242:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768008.497244:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768008.497246:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768009.393222:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768009.393228:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768009.393230:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768009.393231:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768009.393235:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768009.393236:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:3.0:1775768009.393237:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775768009.393239:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768009.393240:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768009.393242:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768009.393244:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:0.0:1775768009.393246:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768009.393249:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768009.393250:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768009.393251:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768009.393255:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768009.393255:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768009.393255:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768009.393261:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768009.393264:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768009.393267:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768009.393267:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768009.393271:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768009.393274:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768009.393275:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768009.393277:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768009.393281:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768009.457253:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768009.457258:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768009.457263:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768009.457265:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768009.457268:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768009.457270:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768009.457273:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768009.457274:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768009.457277:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768009.457278:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768009.457295:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768009.457297:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768009.457300:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768009.521233:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768009.521238:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768009.521241:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768009.521245:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768009.521249:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768009.521252:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768009.521255:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768009.521257:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768009.521259:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768009.713249:0:14953:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4299934526, at 4299934528 00000100:00000040:1.0:1775768009.713249:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299934526, at 4299934528 00000100:00000010:1.0:1775768009.713264:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3840. 00000100:00000010:0.0:1775768009.713266:0:14953:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3d80. 00000100:00000001:3.0:1775768010.417247:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768010.417249:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768010.417249:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768010.417256:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768010.417256:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768010.417256:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768010.417259:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768010.417260:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768010.417260:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768010.417267:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:2.0:1775768010.417267:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775768010.417267:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775768010.417274:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768010.417274:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768010.417274:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768010.417277:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768010.417279:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768010.417279:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768010.417281:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768010.417284:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768010.417288:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768010.417288:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768010.417291:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768010.417293:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768010.417294:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768010.417297:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768010.417300:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768010.481236:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768010.481242:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768010.481248:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768010.481250:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768010.481253:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768010.481254:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768010.481257:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768010.481258:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768010.481261:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768010.481263:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768010.481280:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768010.481281:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768010.481283:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768010.545208:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768010.545213:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768010.545215:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768010.545219:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768010.545221:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768010.545224:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768010.545226:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768010.545228:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768010.545230:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768011.441225:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768011.441230:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768011.441232:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768011.441233:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768011.441234:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768011.441234:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768011.441236:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768011.441239:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:2.0:1775768011.441240:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775768011.441240:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768011.441241:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768011.441246:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768011.441246:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768011.441246:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768011.441250:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768011.441251:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768011.441251:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768011.441254:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768011.441255:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768011.441258:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768011.441260:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768011.441262:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768011.441263:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768011.441265:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768011.441265:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768011.441268:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768011.441270:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768011.505224:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768011.505229:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768011.505231:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768011.505235:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768011.505239:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768011.505242:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768011.505245:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768011.505248:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768011.505251:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768011.505254:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768011.505258:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768011.505260:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768011.505263:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768011.505264:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768011.505267:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768011.505268:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768011.505285:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768011.505287:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768011.505289:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768011.569262:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768011.569268:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768011.569270:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768011.569276:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768011.569279:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768011.569284:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768011.569286:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768011.569288:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768011.569291:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768012.465258:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768012.465258:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768012.465258:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768012.465265:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768012.465265:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768012.465265:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768012.465270:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768012.465270:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768012.465270:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768012.465277:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:2.0:1775768012.465277:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775768012.465277:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768012.465282:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768012.465284:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768012.465284:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768012.465287:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768012.465288:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768012.465289:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768012.465290:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768012.465295:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768012.465298:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768012.465302:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768012.465305:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768012.465307:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768012.465308:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768012.465310:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768012.465313:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768012.529245:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768012.529251:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768012.529256:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768012.529258:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768012.529261:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768012.529263:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768012.529266:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768012.529268:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768012.529271:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768012.529272:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768012.529291:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768012.529293:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768012.529296:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768012.593221:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768012.593227:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768012.593229:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768012.593234:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768012.593238:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768012.593241:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768012.593243:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768012.593245:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768012.593248:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768013.489233:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768013.489235:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768013.489238:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768013.489239:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768013.489242:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768013.489242:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768013.489249:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775768013.489249:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768013.489255:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768013.489255:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768013.489258:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768013.489258:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768013.489263:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768013.489265:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768013.489267:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768013.489268:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768013.489270:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768013.489272:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768013.489284:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768013.489288:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768013.489290:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768013.489291:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775768013.489293:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768013.489297:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:1.0:1775768013.489298:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775768013.489299:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768013.489301:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1775768013.489302:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768013.489305:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768013.489306:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768013.489306:0:8:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768013.489312:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775768013.489319:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5265 00000400:00000200:1.0:1775768013.489328:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768013.489333:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775768013.489336:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775768013.489339:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768013.489342:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768013.489344:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768013.489347:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775768013.489350:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775768013.489352:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:1.0:1775768013.489354:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775768013.489356:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:1.0:1775768013.489359:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:1.0:1775768013.489361:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:1.0:1775768013.489364:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768013.489366:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768013.489368:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768013.489370:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775768013.489372:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:1.0:1775768013.489375:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775768013.489378:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768013.489380:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775768013.489383:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768013.489386:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768013.489388:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775768013.489391:0:8:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775768013.489398:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207889f80. 00000020:00000040:1.0:1775768013.489402:0:8:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:1.0:1775768013.489405:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768013.489407:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768013.489409:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775768013.489412:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775768013.489415:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775768013.489422:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543f000. 00000100:00000001:1.0:1775768013.489428:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768013.489436:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207889f80 x1862022294263040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775768013.489446:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775768013.489450:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768013.489454:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ed2274f4] to pc [ptlrpcd_rcv+-1] req@ffff994207889f80 x1862022294263040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768013.489472:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768013.489474:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768013.489478:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775768013.489479:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775768013.489481:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768013.489484:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775768013.489487:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5265 00000100:00000001:0.0:1775768013.489489:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:1.0:1775768013.489492:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768013.489493:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768013.489496:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000400:00000200:1.0:1775768013.489498:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775768013.489501:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768013.489503:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207889f80 x1862022294263040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775768013.489506:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768013.489511:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768013.489515:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768013.489517:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768013.489519:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:0.0:1775768013.489520:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768013.489524:0:8:0:(genops.c:811:class_conn2export()) Process entered 02000000:00000001:0.0:1775768013.489525:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:1.0:1775768013.489526:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 02000000:00000001:0.0:1775768013.489528:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1775768013.489529:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775768013.489531:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 02000000:00000001:0.0:1775768013.489532:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768013.489535:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00100000:0.0:1775768013.489538:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207889f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294263040:192.168.202.156@tcp:250:kworker.0 00000020:00000001:1.0:1775768013.489539:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000001:0.0:1775768013.489544:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000040:1.0:1775768013.489545:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 02000000:00000001:0.0:1775768013.489547:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1775768013.489548:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1775768013.489551:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768013.489552:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000001:0.0:1775768013.489553:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1775768013.489555:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 02000000:00000010:0.0:1775768013.489557:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002b800. 00000020:00000040:1.0:1775768013.489558:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 02000000:00000001:0.0:1775768013.489562:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768013.489563:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775768013.489566:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768013.489570:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:1.0:1775768013.489570:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:0.0:1775768013.489574:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 02000000:00000001:1.0:1775768013.489577:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:0.0:1775768013.489579:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 02000000:00000001:1.0:1775768013.489580:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768013.489583:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000200:0.0:1775768013.489585:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294263040, portal 25 00000008:00100000:1.0:1775768013.489587:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1775768013.489589:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768013.489592:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606126464 : -112966103425152 : ffff994207889f80) 00000008:00000001:1.0:1775768013.489592:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768013.489596:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768013.489600:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207889f80 x1862022294263040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768038 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1775768013.489602:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff99420788b100. 00000020:00000040:1.0:1775768013.489606:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000100:00000001:0.0:1775768013.489611:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:1.0:1775768013.489611:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768013.489613:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768013.489615:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000100:00000001:1.0:1775768013.489616:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775768013.489618:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:0.0:1775768013.489619:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 02000000:00000001:1.0:1775768013.489620:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1775768013.489621:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294263040, offset 0 02000000:00000010:1.0:1775768013.489625:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543c800. 00000400:00000200:0.0:1775768013.489629:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00000001:1.0:1775768013.489630:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768013.489635:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00100000:1.0:1775768013.489637:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff99420788b100 x1862022294263168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1775768013.489641:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768013.489645:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768013.489648:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768013.489651:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768013.489654:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768013.489661:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225228:225228:256:4294967295] 192.168.202.156@tcp LPNI seq info [225224:225224:8:4294967295] 00000100:00000001:1.0:1775768013.489663:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:0.0:1775768013.489667:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768013.489667:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768013.489673:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e26cf977] to pc [ptlrpcd_rcv+-1] req@ffff99420788b100 x1862022294263168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775768013.489681:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000100:00000001:1.0:1775768013.489693:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1775768013.489694:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00080000:1.0:1775768013.489697:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000010:0.0:1775768013.489700:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000100:00080000:1.0:1775768013.489701:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775768013.489704:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775768013.489705:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768013.489709:0:8:0:(import.c:484:import_select_connection()) Process entered 00000800:00000200:0.0:1775768013.489710:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00080000:1.0:1775768013.489714:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5265 00000800:00000200:0.0:1775768013.489716:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000400:00000200:1.0:1775768013.489720:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768013.489726:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775768013.489728:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775768013.489731:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768013.489735:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:0.1:1775768013.489735:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000040:1.0:1775768013.489739:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000800:00000200:0.1:1775768013.489740:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000100:00000001:1.0:1775768013.489743:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775768013.489747:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000800:00000100:0.0:1775768013.489748:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000020:00000040:1.0:1775768013.489749:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000800:00000010:0.0:1775768013.489752:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000020:00000001:1.0:1775768013.489752:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775768013.489754:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:1.0:1775768013.489757:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:1.0:1775768013.489759:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000400:00000200:0.0:1775768013.489760:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:1.0:1775768013.489775:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768013.489778:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1775768013.489782:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000100:00000040:1.0:1775768013.489784:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768013.489787:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000200:0.0:1775768013.489790:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000020:00000040:1.0:1775768013.489790:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:1.0:1775768013.489794:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000400:00000010:0.0:1775768013.489795:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:1.0:1775768013.489798:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768013.489801:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1775768013.489803:0:2399:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:1.0:1775768013.489804:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768013.489807:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1775768013.489809:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207889f80 x1862022294263040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768038 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768013.489810:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775768013.489813:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775768013.489818:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768013.489821:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775768013.489827:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207888700. 00000100:00000001:0.0:1775768013.489829:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000020:00000040:1.0:1775768013.489831:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000100:00000040:0.0:1775768013.489833:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207889f80 x1862022294263040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768038 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1775768013.489834:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768013.489837:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768013.489840:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775768013.489843:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1775768013.489844:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768013.489846:0:2399:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:1.0:1775768013.489846:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775768013.489851:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543e800. 00000100:00000001:0.0:1775768013.489852:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768013.489854:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768013.489858:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768013.489860:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299939645 00000100:00100000:1.0:1775768013.489863:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207888700 x1862022294263296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000010:0.0:1775768013.489866:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942078a3d80. 00000100:00000001:0.0:1775768013.489873:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775768013.489878:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299939645 00000100:00000010:0.0:1775768013.489883:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942078a32a0. 00000100:00000001:1.0:1775768013.489883:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768013.489886:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775768013.489886:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768013.489889:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1775768013.489892:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009f2ba301] to pc [ptlrpcd_rcv+-1] req@ffff994207888700 x1862022294263296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768013.489893:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768013.489897:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768013.489903:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768013/real 1775768013] req@ffff994207889f80 x1862022294263040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768013.489907:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768013.489912:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000001:0.0:1775768013.489916:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:1.0:1775768013.489916:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000400:00000001:0.0:1775768013.489919:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1775768013.489919:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000400:00000010:0.0:1775768013.489921:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000100:00080000:1.0:1775768013.489922:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:0.0:1775768013.489925:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000100:00000001:1.0:1775768013.489926:0:8:0:(import.c:484:import_select_connection()) Process entered 00000400:00000010:0.0:1775768013.489927:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00080000:1.0:1775768013.489930:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5265 00000100:00000001:0.0:1775768013.489932:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000200:1.0:1775768013.489936:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000200:0.0:1775768013.489939:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207889f80 x1862022294263040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775768013.489942:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775768013.489946:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775768013.489951:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768013.489955:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768013.489960:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000200:0.0:1775768013.489961:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207889f80 x1862022294263040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768013.489964:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775768013.489968:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775768013.489971:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:1.0:1775768013.489974:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:0.0:1775768013.489977:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000040:1.0:1775768013.489977:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000100:00000001:0.0:1775768013.489981:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1775768013.489981:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:1.0:1775768013.489985:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:0.0:1775768013.489986:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768013.489988:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768013.489991:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00100000:0.0:1775768013.489995:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207889f80 x1862022294263040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768013.489995:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768013.490000:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768013.490005:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:0.0:1775768013.490009:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:1.0:1775768013.490010:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:1.0:1775768013.490015:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000040:0.0:1775768013.490016:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207889f80 x1862022294263040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768038 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768013.490018:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768013.490022:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775768013.490026:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768013.490027:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768013.490030:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1775768013.490030:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768013.490035:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768013.490035:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775768013.490038:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:1.0:1775768013.490039:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000100:00000001:0.0:1775768013.490040:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768013.490042:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00000001:1.0:1775768013.490043:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768013.490045:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:1.0:1775768013.490046:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768013.490050:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768013.490051:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000010:1.0:1775768013.490051:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207889880. 00000020:00000040:1.0:1775768013.490056:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00000100:00000040:0.0:1775768013.490057:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207889f80 x1862022294263040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768038 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1775768013.490060:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768013.490063:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768013.490066:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775768013.490068:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:0.0:1775768013.490072:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207889f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294263040:192.168.202.156@tcp:250:kworker.0 02000000:00000001:1.0:1775768013.490072:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768013.490077:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:1.0:1775768013.490077:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543f800. 00000100:00000001:0.0:1775768013.490081:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1775768013.490082:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768013.490084:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207889f80 x1862022294263040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768038 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768013.490091:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 00000100:00100000:1.0:1775768013.490091:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207889880 x1862022294263424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1775768013.490093:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768013.490096:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002b800. 02000000:00000001:0.0:1775768013.490100:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768013.490102:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:1.0:1775768013.490103:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000020:00000040:0.0:1775768013.490104:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:0.0:1775768013.490107:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1775768013.490107:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768013.490110:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543f000. 00000100:00000040:1.0:1775768013.490113:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f11856b8] to pc [ptlrpcd_rcv+-1] req@ffff994207889880 x1862022294263424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1775768013.490114:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768013.490117:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768013.490119:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207889f80. 00000100:00000001:0.0:1775768013.490124:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768013.490126:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768013.490128:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768013.490129:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768013.490132:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775768013.490134:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:1.0:1775768013.490135:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775768013.490136:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768013.490138:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1775768013.490139:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775768013.490141:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:1.0:1775768013.490141:0:8:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768013.490144:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1775768013.490146:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00000040:0.0:1775768013.490150:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff99420788b100 x1862022294263168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768013.490152:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5265 00000100:00000001:0.0:1775768013.490173:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768013.490174:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768013.490176:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768013.490177:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768013.490180:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768013.490181:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00100000:0.0:1775768013.490185:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff99420788b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294263168:192.168.202.156@tcp:38:kworker.0 00000400:00000200:1.0:1775768013.490187:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:0.0:1775768013.490190:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00080000:1.0:1775768013.490191:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 02000000:00000001:0.0:1775768013.490194:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:1.0:1775768013.490196:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 02000000:00000001:0.0:1775768013.490197:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768013.490199:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1775768013.490201:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:1.0:1775768013.490203:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000010:0.0:1775768013.490206:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002b800. 00000100:00000001:1.0:1775768013.490206:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 02000000:00000001:0.0:1775768013.490210:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768013.490211:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775768013.490213:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:1.0:1775768013.490215:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775768013.490217:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000400:00000200:0.0:1775768013.490219:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:1.0:1775768013.490220:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:1.0:1775768013.490223:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000400:00000010:0.0:1775768013.490224:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000100:00000040:1.0:1775768013.490228:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000010:0.0:1775768013.490230:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000001:1.0:1775768013.490230:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768013.490234:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000200:0.0:1775768013.490236:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294263168, portal 10 00000100:00000001:1.0:1775768013.490237:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:0.0:1775768013.490240:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000020:00000040:1.0:1775768013.490241:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00000001:0.0:1775768013.490243:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606130944 : -112966103420672 : ffff99420788b100) 00000100:00080000:1.0:1775768013.490245:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775768013.490248:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768013.490251:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff99420788b100 x1862022294263168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768038 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1775768013.490251:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775768013.490255:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768013.490259:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768013.490263:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775768013.490264:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00100000:1.0:1775768013.490265:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000100:00000040:0.0:1775768013.490268:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000008:00000001:1.0:1775768013.490271:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775768013.490272:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000001:1.0:1775768013.490274:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1775768013.490275:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294263168, offset 0 00000100:00000010:1.0:1775768013.490278:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207888e00. 00000400:00000200:0.0:1775768013.490281:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000020:00000040:1.0:1775768013.490282:0:8:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00000400:00000200:0.0:1775768013.490286:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:1.0:1775768013.490286:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768013.490289:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768013.490291:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00000001:1.0:1775768013.490292:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:0.0:1775768013.490295:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:1.0:1775768013.490295:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775768013.490297:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768013.490299:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768013.490302:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 02000000:00000010:1.0:1775768013.490303:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543c400. 00000400:00000200:0.0:1775768013.490304:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768013.490308:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225229:225229:256:4294967295] 192.168.202.156@tcp LPNI seq info [225225:225225:8:4294967295] 00000100:00000001:1.0:1775768013.490308:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768013.490313:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00100000:1.0:1775768013.490316:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207888e00 x1862022294263552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1775768013.490321:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768013.490331:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00000001:1.0:1775768013.490335:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000010:0.0:1775768013.490336:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000100:00000001:1.0:1775768013.490338:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1775768013.490342:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768013.490346:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000100:00000040:1.0:1775768013.490346:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001812f966] to pc [ptlrpcd_rcv+-1] req@ffff994207888e00 x1862022294263552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.1:1775768013.490354:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768013.490359:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1775768013.490366:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000100:00000001:1.0:1775768013.490366:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1775768013.490369:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775768013.490372:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:1.0:1775768013.490372:0:8:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5276) 00000400:00000200:0.0:1775768013.490378:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768013.490379:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775768013.490381:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775768013.490382:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768013.490385:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff99420788b100 x1862022294263168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768038 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768013.490391:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768013.490393:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff99420788b100 x1862022294263168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768038 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768013.490401:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768013.490402:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768013.490404:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768013.490407:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:0.0:1775768013.490409:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768013.490411:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299938323 00000100:00000001:0.0:1775768013.490412:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299939645 00000100:00000001:0.0:1775768013.490414:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299938323 00000100:00000001:0.0:1775768013.490415:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299939645 00000100:00000001:0.0:1775768013.490416:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768013.490418:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768013.490419:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768013.490421:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768013/real 1775768013] req@ffff99420788b100 x1862022294263168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768013.490426:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768013.490427:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768013.490428:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775768013.490429:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768013.490430:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768013.490432:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768013.490433:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff99420788b100 x1862022294263168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768013.490438:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff99420788b100 x1862022294263168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768013.490441:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768013.490442:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768013.490443:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768013.490444:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768013.490446:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff99420788b100 x1862022294263168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768013.490450:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768013.490453:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff99420788b100 x1862022294263168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768038 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768013.490456:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768013.490458:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768013.490459:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768013.490459:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768013.490460:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768013.490462:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768013.490464:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768013.490466:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768013.490467:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768013.490469:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff99420788b100 x1862022294263168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768038 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768013.490473:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff99420788b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294263168:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768013.490475:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768013.490477:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207888700 x1862022294263296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768013.490483:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768013.490484:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768013.490486:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768013.490488:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768013.490490:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768013.490494:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207888700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294263296:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768013.490497:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768013.490499:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768013.490501:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768013.490503:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768013.490507:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220029800. 02000000:00000001:0.0:1775768013.490509:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768013.490512:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768013.490516:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775768013.490520:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768013.490524:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294263296, portal 10 00000100:00000001:0.0:1775768013.490526:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768013.490528:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606120192 : -112966103431424 : ffff994207888700) 00000100:00000040:0.0:1775768013.490534:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207888700 x1862022294263296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768038 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768013.490541:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768013.490544:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768013.490546:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775768013.490548:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294263296, offset 0 00000400:00000200:0.0:1775768013.490553:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768013.490556:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768013.490559:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768013.490562:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768013.490564:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768013.490567:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768013.490570:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768013.490573:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225230:225230:256:4294967295] 192.168.202.156@tcp LPNI seq info [225226:225226:8:4294967295] 00000400:00000200:0.0:1775768013.490577:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768013.490583:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768013.490587:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768013.490590:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775768013.490594:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768013.490596:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775768013.490601:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768013.490603:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1775768013.490607:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768013.490609:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775768013.490612:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768013.490616:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768013.490619:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775768013.490621:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775768013.490624:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768013.490629:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207888700 x1862022294263296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768038 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768013.490637:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768013.490640:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207888700 x1862022294263296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768038 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768013.490646:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768013.490648:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768013.490652:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768013.490655:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:0.0:1775768013.490658:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768013.490659:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299938315 00000100:00000001:0.0:1775768013.490662:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299939645 00000100:00000001:0.0:1775768013.490663:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299938315 00000100:00000001:0.0:1775768013.490665:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299939645 00000100:00000001:0.0:1775768013.490666:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768013.490668:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768013.490669:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768013.490672:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768013/real 1775768013] req@ffff994207888700 x1862022294263296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768013.490676:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768013.490677:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768013.490678:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775768013.490679:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768013.490680:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768013.490682:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768013.490685:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207888700 x1862022294263296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768013.490693:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207888700 x1862022294263296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768013.490699:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768013.490700:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768013.490702:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768013.490703:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768013.490706:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207888700 x1862022294263296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768013.490713:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768013.490717:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207888700 x1862022294263296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768038 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768013.490725:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768013.490727:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768013.490728:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768013.490729:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768013.490731:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768013.490733:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768013.490735:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768013.490738:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768013.490740:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768013.490742:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207888700 x1862022294263296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768038 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768013.490750:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207888700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294263296:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768013.490755:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768013.490759:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207889880 x1862022294263424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768013.490777:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768013.490779:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768013.490781:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768013.490783:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768013.490785:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768013.490789:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207889880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294263424:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768013.490792:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768013.490794:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768013.490795:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768013.490798:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768013.490802:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220029400. 02000000:00000001:0.0:1775768013.490805:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768013.490808:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768013.490810:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775768013.490813:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768013.490815:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294263424, portal 4 00000100:00000001:0.0:1775768013.490817:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768013.490818:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606124672 : -112966103426944 : ffff994207889880) 00000100:00000040:0.0:1775768013.490822:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207889880 x1862022294263424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768038 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768013.490826:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768013.490829:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768013.490831:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775768013.490833:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294263424, offset 0 00000400:00000200:0.0:1775768013.490837:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768013.490840:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768013.490843:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768013.490846:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768013.490849:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768013.490851:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768013.490853:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768013.490857:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225231:225231:256:4294967295] 192.168.202.156@tcp LPNI seq info [225227:225227:8:4294967295] 00000400:00000200:0.0:1775768013.490860:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768013.490864:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768013.490866:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768013.490868:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775768013.490871:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768013.490873:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775768013.490876:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768013.490878:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1775768013.490880:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768013.490882:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775768013.490883:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768013.490886:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768013.490887:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775768013.490888:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775768013.490890:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768013.490892:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207889880 x1862022294263424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768038 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768013.490897:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768013.490899:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207889880 x1862022294263424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768038 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768013.490903:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768013.490904:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768013.490905:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768013.490907:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:0.0:1775768013.490909:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768013.490910:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299938338 00000100:00000001:0.0:1775768013.490912:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299939645 00000100:00000001:0.0:1775768013.490913:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299938338 00000100:00000001:0.0:1775768013.490915:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299939645 00000100:00000001:0.0:1775768013.490916:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768013.490918:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768013.490919:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768013.490921:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768013/real 1775768013] req@ffff994207889880 x1862022294263424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768013.490926:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768013.490927:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768013.490928:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775768013.490930:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768013.490931:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768013.490932:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768013.490934:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207889880 x1862022294263424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768013.490940:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207889880 x1862022294263424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768013.490944:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768013.490944:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768013.490945:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768013.490946:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768013.490948:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207889880 x1862022294263424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768013.490952:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768013.490955:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207889880 x1862022294263424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768038 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768013.490959:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768013.490960:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768013.490961:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768013.490961:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768013.490963:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768013.490965:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768013.490966:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768013.490969:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768013.490971:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768013.490974:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207889880 x1862022294263424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768038 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768013.490978:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207889880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294263424:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768013.490981:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768013.490982:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768013.490984:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff99420788b100 x1862022294263168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768038 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768013.490988:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768013.490989:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768013.490990:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002b800. 02000000:00000001:0.0:1775768013.490993:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768013.490994:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768013.490995:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:0.0:1775768013.490996:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768013.490997:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543c800. 02000000:00000001:0.0:1775768013.490999:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768013.491000:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768013.491001:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff99420788b100. 00000100:00000001:0.0:1775768013.491002:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768013.491003:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768013.491004:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768013.491006:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207888700 x1862022294263296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768038 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768013.491009:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768013.491010:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768013.491011:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220029800. 02000000:00000001:0.0:1775768013.491012:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768013.491012:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768013.491013:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:0.0:1775768013.491014:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768013.491015:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543e800. 02000000:00000001:0.0:1775768013.491016:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768013.491017:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768013.491018:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207888700. 00000100:00000001:0.0:1775768013.491019:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768013.491020:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768013.491021:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768013.491022:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207889880 x1862022294263424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768038 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768013.491025:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768013.491026:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768013.491026:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220029400. 02000000:00000001:0.0:1775768013.491027:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768013.491028:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768013.491028:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:0.0:1775768013.491029:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768013.491030:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543f800. 02000000:00000001:0.0:1775768013.491031:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768013.491031:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768013.491032:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207889880. 00000100:00000001:0.0:1775768013.491033:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768013.491034:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768013.491035:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768013.491037:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768013.491038:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768013.491039:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775768013.491040:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768013.491040:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768013.491042:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207888e00 x1862022294263552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768013.491045:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768013.491047:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768013.491048:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768013.491049:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768013.491050:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768013.491053:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207888e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294263552:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768013.491055:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768013.491056:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768013.491057:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768013.491058:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768013.491059:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220029400. 02000000:00000001:0.0:1775768013.491061:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768013.491063:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768013.491064:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775768013.491066:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768013.491067:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294263552, portal 4 00000100:00000001:0.0:1775768013.491069:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768013.491069:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606121984 : -112966103429632 : ffff994207888e00) 00000100:00000040:0.0:1775768013.491073:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207888e00 x1862022294263552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768038 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768013.491076:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768013.491078:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768013.491079:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775768013.491080:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294263552, offset 0 00000400:00000200:0.0:1775768013.491082:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768013.491084:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768013.491085:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768013.491087:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768013.491088:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768013.491089:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768013.491091:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768013.491093:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225232:225232:256:4294967295] 192.168.202.156@tcp LPNI seq info [225228:225228:8:4294967295] 00000400:00000200:0.0:1775768013.491095:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768013.491098:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768013.491100:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768013.491103:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775768013.491106:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768013.491108:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775768013.491110:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768013.491112:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1775768013.491114:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768013.491115:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775768013.491117:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768013.491119:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768013.491121:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775768013.491121:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775768013.491123:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768013.491124:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207888e00 x1862022294263552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768038 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768013.491129:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768013.491130:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207888e00 x1862022294263552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768038 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768013.491133:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768013.491134:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768013.491135:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768013.491137:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:0.0:1775768013.491138:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768013.491139:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299938341 00000100:00000001:0.0:1775768013.491141:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299939645 00000100:00000001:0.0:1775768013.491142:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299938341 00000100:00000001:0.0:1775768013.491143:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299939645 00000100:00000001:0.0:1775768013.491144:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768013.491145:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768013.491146:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768013.491148:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768013/real 1775768013] req@ffff994207888e00 x1862022294263552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768013.491152:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768013.491152:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768013.491170:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775768013.491173:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768013.491174:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768013.491176:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768013.491178:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207888e00 x1862022294263552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768013.491184:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207888e00 x1862022294263552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768013.491189:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768013.491190:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768013.491192:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768013.491194:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768013.491197:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207888e00 x1862022294263552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768013.491203:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768013.491207:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207888e00 x1862022294263552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768038 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768013.491214:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768013.491216:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768013.491218:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768013.491220:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768013.491221:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768013.491225:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768013.491228:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768013.491232:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768013.491235:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768013.491238:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207888e00 x1862022294263552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768038 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768013.491245:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207888e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294263552:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768013.491248:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768013.491249:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768013.491251:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207888e00 x1862022294263552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768038 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768013.491254:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768013.491255:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768013.491256:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220029400. 02000000:00000001:0.0:1775768013.491257:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768013.491258:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768013.491259:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:0.0:1775768013.491260:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768013.491261:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543c400. 02000000:00000001:0.0:1775768013.491262:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768013.491263:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768013.491264:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207888e00. 00000100:00000001:0.0:1775768013.491265:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768013.491266:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768013.491267:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768013.491268:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768013.491269:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768013.491270:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775768013.491271:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768013.553237:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768013.553243:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768013.553248:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768013.553249:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768013.553251:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768013.553252:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768013.553254:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768013.553255:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768013.553257:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768013.553257:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768013.553270:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768013.553272:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768013.553274:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768013.617248:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768013.617253:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768013.617254:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768013.617258:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768013.617261:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768013.617264:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768013.617265:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768013.617267:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768013.617269:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768014.513219:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768014.513219:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768014.513220:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768014.513225:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768014.513226:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768014.513226:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768014.513227:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768014.513229:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768014.513230:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768014.513233:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:2.0:1775768014.513236:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775768014.513236:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775768014.513237:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768014.513240:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768014.513241:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768014.513242:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768014.513243:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768014.513246:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768014.513246:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768014.513247:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768014.513253:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768014.513253:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768014.513256:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768014.513260:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768014.513260:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768014.513263:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768014.513267:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768014.577238:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768014.577244:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768014.577248:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768014.577249:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768014.577253:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768014.577254:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768014.577257:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768014.577258:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768014.577260:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768014.577261:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768014.577277:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768014.577279:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768014.577281:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768014.641254:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768014.641259:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768014.641261:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768014.641265:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768014.641267:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768014.641270:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768014.641272:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768014.641274:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768014.641276:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768014.833248:0:14953:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4299939645, at 4299939648 00000100:00000040:1.0:1775768014.833249:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299939645, at 4299939648 00000100:00000010:0.0:1775768014.833260:0:14953:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3d80. 00000100:00000010:1.0:1775768014.833263:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a32a0. 00000100:00000001:3.0:1775768015.537256:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768015.537262:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768015.537263:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768015.537265:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768015.537271:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768015.537271:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768015.537272:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768015.537275:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768015.537275:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768015.537286:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:0.0:1775768015.537287:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:2.0:1775768015.537287:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:0.0:1775768015.537299:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768015.537299:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768015.537299:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768015.537309:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768015.537310:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768015.537310:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768015.537311:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768015.537316:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768015.537321:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768015.537322:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768015.537325:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768015.537330:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768015.537331:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768015.537334:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768015.537339:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768015.601243:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768015.601248:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768015.601253:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768015.601256:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768015.601257:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768015.601259:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768015.601261:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768015.601263:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768015.601264:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768015.601266:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768015.601268:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768015.601269:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768015.601270:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768015.601271:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768015.601273:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768015.601276:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768015.601277:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768015.601279:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768015.601282:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768015.601284:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768015.601284:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768015.601285:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768015.601286:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768015.601288:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768015.601289:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768015.601292:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768015.601294:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768015.601295:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768015.601296:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768015.601297:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768015.601298:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768015.601298:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768015.601300:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768015.601302:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768015.601321:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768015.601322:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768015.601324:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768015.665246:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768015.665252:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768015.665253:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768015.665257:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768015.665259:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768015.665262:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768015.665264:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768015.665265:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768015.665267:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768016.561231:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768016.561235:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768016.561238:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768016.561240:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768016.561241:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768016.561243:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768016.561247:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775768016.561247:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768016.561253:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768016.561254:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768016.561260:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768016.561260:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768016.561262:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768016.561266:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768016.561267:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768016.561271:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768016.561271:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768016.561280:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768016.561281:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768016.561282:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775768016.561283:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768016.561292:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768016.561293:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768016.561301:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768016.561305:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768016.561306:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768016.561309:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768016.625269:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768016.625274:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768016.625277:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768016.625278:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768016.625280:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768016.625281:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768016.625283:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768016.625283:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768016.625285:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768016.625286:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768016.625303:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768016.625305:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768016.625306:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768016.689271:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768016.689278:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768016.689280:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768016.689285:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768016.689289:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768016.689293:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768016.689295:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768016.689297:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768016.689300:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768017.585216:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768017.585225:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768017.585227:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768017.585235:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768017.585240:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768017.585245:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768017.585246:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768017.585249:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768017.585251:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768017.585253:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768017.585254:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768017.585258:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768017.585259:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768017.585265:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768017.585268:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768017.585268:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768017.585273:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768017.585274:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768017.585276:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768017.585277:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768017.585282:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768017.585283:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775768017.585289:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768017.585292:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768017.585294:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768017.585296:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768017.585299:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768017.649264:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768017.649271:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768017.649279:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768017.649281:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768017.649286:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768017.649287:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768017.649291:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768017.649293:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768017.649296:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768017.649298:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768017.649322:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768017.649325:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768017.649328:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768017.713318:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768017.713327:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768017.713329:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768017.713336:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768017.713341:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768017.713346:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768017.713349:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768017.713353:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768017.713356:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768018.609220:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768018.609228:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768018.609231:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768018.609237:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:3.0:1775768018.609238:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768018.609243:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768018.609244:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768018.609247:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768018.609250:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768018.609251:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:0.0:1775768018.609252:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768018.609254:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768018.609256:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768018.609258:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768018.609258:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768018.609260:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768018.609262:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768018.609263:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768018.609263:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768018.609267:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768018.609270:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768018.609271:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768018.609276:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768018.609279:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768018.609281:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768018.609284:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1775768018.609286:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775768018.609287:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768018.609298:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775768018.609302:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775768018.609306:0:8:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768018.609311:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775768018.609318:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5271 00000400:00000200:1.0:1775768018.609328:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768018.609334:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775768018.609336:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775768018.609339:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768018.609341:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768018.609343:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768018.609344:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775768018.609347:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775768018.609349:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:1.0:1775768018.609351:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775768018.609353:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:1.0:1775768018.609355:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:1.0:1775768018.609356:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:1.0:1775768018.609358:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768018.609359:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768018.609360:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768018.609361:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775768018.609362:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:1.0:1775768018.609365:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775768018.609366:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768018.609368:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775768018.609371:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768018.609372:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768018.609374:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775768018.609375:0:8:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775768018.609382:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207888e00. 00000020:00000040:1.0:1775768018.609384:0:8:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:1.0:1775768018.609386:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768018.609387:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768018.609389:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775768018.609391:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775768018.609394:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775768018.609402:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543d800. 00000100:00000001:1.0:1775768018.609406:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768018.609413:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207888e00 x1862022294263680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775768018.609419:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775768018.609423:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768018.609426:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001812f966] to pc [ptlrpcd_rcv+-1] req@ffff994207888e00 x1862022294263680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768018.609448:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768018.609450:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768018.609453:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775768018.609453:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775768018.609455:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768018.609456:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775768018.609458:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5271 00000400:00000200:1.0:1775768018.609461:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768018.609464:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775768018.609466:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775768018.609470:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768018.609472:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768018.609473:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768018.609476:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768018.609476:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:0.0:1775768018.609481:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:1.0:1775768018.609481:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775768018.609483:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000100:00000001:0.0:1775768018.609485:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000001:1.0:1775768018.609485:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775768018.609490:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:1.0:1775768018.609492:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:1.0:1775768018.609496:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:0.0:1775768018.609500:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207888e00 x1862022294263680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775768018.609505:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768018.609512:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768018.609523:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768018.609532:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768018.609534:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:0.0:1775768018.609535:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768018.609541:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:1.0:1775768018.609541:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 02000000:00000001:0.0:1775768018.609546:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:1.0:1775768018.609549:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 02000000:00000001:0.0:1775768018.609550:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768018.609557:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768018.609558:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207888e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294263680:192.168.202.156@tcp:250:kworker.0 02000000:00000001:1.0:1775768018.609568:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1775768018.609569:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768018.609573:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768018.609575:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768018.609579:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768018.609579:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768018.609587:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:0.0:1775768018.609588:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:1.0:1775768018.609594:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 02000000:00000010:0.0:1775768018.609596:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002b800. 00000008:00000001:1.0:1775768018.609597:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768018.609602:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768018.609603:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775768018.609608:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207889880. 00000020:00000040:1.0:1775768018.609610:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000400:00000200:0.0:1775768018.609617:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:1.0:1775768018.609618:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768018.609621:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775768018.609623:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000100:00000001:1.0:1775768018.609623:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775768018.609629:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:0.0:1775768018.609633:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 02000000:00000001:1.0:1775768018.609638:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1775768018.609643:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294263680, portal 25 02000000:00000010:1.0:1775768018.609643:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543e400. 00000100:00000001:0.0:1775768018.609653:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768018.609654:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768018.609656:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606121984 : -112966103429632 : ffff994207888e00) 00000100:00100000:1.0:1775768018.609665:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207889880 x1862022294263808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:0.0:1775768018.609671:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207888e00 x1862022294263680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768043 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768018.609683:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775768018.609687:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768018.609700:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f11856b8] to pc [ptlrpcd_rcv+-1] req@ffff994207889880 x1862022294263808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768018.609702:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768018.609727:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000100:00000001:1.0:1775768018.609729:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768018.609731:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:0.0:1775768018.609732:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00080000:1.0:1775768018.609735:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775768018.609737:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775768018.609739:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:0.0:1775768018.609740:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294263680, offset 0 00000100:00000001:1.0:1775768018.609742:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775768018.609745:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5271 00000400:00000200:1.0:1775768018.609750:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768018.609754:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000400:00000200:0.0:1775768018.609759:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00080000:1.0:1775768018.609759:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775768018.609772:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000200:0.0:1775768018.609775:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768018.609780:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1775768018.609783:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00000040:1.0:1775768018.609784:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000200:0.0:1775768018.609790:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1775768018.609791:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000200:0.0:1775768018.609798:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000020:00000001:1.0:1775768018.609800:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775768018.609803:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000400:00000200:0.0:1775768018.609804:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768018.609808:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000020:00000001:1.0:1775768018.609810:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775768018.609815:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000400:00000200:0.0:1775768018.609816:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225233:225233:256:4294967295] 192.168.202.156@tcp LPNI seq info [225229:225229:8:4294967295] 00000020:00000001:1.0:1775768018.609826:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000400:00000200:0.0:1775768018.609830:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:1.0:1775768018.609835:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000400:00000200:0.0:1775768018.609849:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768018.609855:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00000040:1.0:1775768018.609858:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000800:00000010:0.0:1775768018.609861:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000100:00000001:1.0:1775768018.609868:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768018.609873:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000800:00000200:0.0:1775768018.609874:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768018.609881:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000100:00000001:1.0:1775768018.609882:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775768018.609893:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000800:00000200:0.1:1775768018.609894:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768018.609902:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000100:00080000:1.0:1775768018.609908:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000800:00000100:0.0:1775768018.609916:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000100:00000001:1.0:1775768018.609917:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1775768018.609919:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 02000000:00000001:1.0:1775768018.609925:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:0.0:1775768018.609927:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000001:1.0:1775768018.609930:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1775768018.609935:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 02000000:00000001:1.0:1775768018.609940:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768018.609943:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000100:00000001:1.0:1775768018.609945:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000010:0.0:1775768018.609946:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000008:00100000:1.0:1775768018.609950:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1775768018.609951:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000008:00000001:1.0:1775768018.609957:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1775768018.609958:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207888e00 x1862022294263680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768043 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768018.609969:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775768018.609982:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207888700. 00000100:00000001:0.0:1775768018.609985:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000020:00000040:1.0:1775768018.609987:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000100:00000040:0.0:1775768018.609990:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207888e00 x1862022294263680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768043 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1775768018.609999:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768018.610005:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768018.610013:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1775768018.610016:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768018.610016:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1775768018.610018:0:2399:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:1.0:1775768018.610021:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768018.610024:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775768018.610028:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543dc00. 00000400:00000200:0.0:1775768018.610031:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768018.610037:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299944765 00000100:00000001:1.0:1775768018.610037:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768018.610044:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942078a3d80. 00000100:00100000:1.0:1775768018.610050:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207888700 x1862022294263936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768018.610055:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775768018.610065:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299944765 00000100:00000010:0.0:1775768018.610072:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942078a3b40. 00000100:00000001:0.0:1775768018.610081:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775768018.610088:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768018.610092:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768018.610093:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768018.610098:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000001:1.0:1775768018.610099:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768018.610106:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768018/real 1775768018] req@ffff994207888e00 x1862022294263680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775768018.610111:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009f2ba301] to pc [ptlrpcd_rcv+-1] req@ffff994207888700 x1862022294263936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768018.610143:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1775768018.610144:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1775768018.610147:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:1.0:1775768018.610149:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:0.0:1775768018.610152:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000100:00080000:1.0:1775768018.610154:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000400:00000200:0.0:1775768018.610192:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768018.610195:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768018.610199:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1775768018.610199:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775768018.610204:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:0.0:1775768018.610208:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207888e00 x1862022294263680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768018.610210:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775768018.610219:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5271 00000100:00000200:0.0:1775768018.610226:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207888e00 x1862022294263680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775768018.610235:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768018.610248:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:1.0:1775768018.610249:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:0.0:1775768018.610254:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1775768018.610256:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1775768018.610260:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768018.610263:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768018.610263:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00100000:0.0:1775768018.610270:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207888e00 x1862022294263680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768018.610271:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768018.610282:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768018.610287:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768018.610289:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000040:0.0:1775768018.610294:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207888e00 x1862022294263680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768043 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1775768018.610295:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775768018.610300:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:1.0:1775768018.610307:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:0.0:1775768018.610308:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000040:1.0:1775768018.610313:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000100:00000001:0.0:1775768018.610314:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1775768018.610321:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:1.0:1775768018.610324:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:0.0:1775768018.610328:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:1.0:1775768018.610328:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768018.610333:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1775768018.610334:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768018.610339:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768018.610341:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768018.610346:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768018.610347:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00080000:0.0:1775768018.610352:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000020:00000040:1.0:1775768018.610353:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:0.0:1775768018.610363:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00080000:1.0:1775768018.610365:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768018.610368:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:1.0:1775768018.610376:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768018.610378:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207888e00 x1862022294263680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768043 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1775768018.610384:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775768018.610393:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1775768018.610394:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207888e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294263680:192.168.202.156@tcp:250:kworker.0 02000000:00000001:1.0:1775768018.610405:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768018.610406:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768018.610412:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1775768018.610413:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000040:0.0:1775768018.610418:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207888e00 x1862022294263680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768043 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1775768018.610419:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:1.0:1775768018.610434:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768018.610438:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 00000100:00000001:1.0:1775768018.610439:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768018.610441:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768018.610447:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002b800. 00000100:00000010:1.0:1775768018.610448:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff99420788b100. 02000000:00000001:0.0:1775768018.610450:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768018.610456:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768018.610457:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000040:0.0:1775768018.610460:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:0.0:1775768018.610465:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000001:1.0:1775768018.610467:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000010:0.0:1775768018.610468:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543d800. 02000000:00000001:1.0:1775768018.610471:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768018.610474:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:1.0:1775768018.610475:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768018.610479:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:1.0:1775768018.610480:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000010:0.0:1775768018.610482:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207888e00. 02000000:00000001:1.0:1775768018.610483:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768018.610489:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 02000000:00000010:1.0:1775768018.610490:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543fc00. 00000100:00000001:0.0:1775768018.610491:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768018.610496:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768018.610498:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768018.610503:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:1.0:1775768018.610503:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff99420788b100 x1862022294264064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768018.610506:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768018.610513:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775768018.610519:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768018.610521:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768018.610527:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207889880 x1862022294263808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768018.610528:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775768018.610535:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768018.610546:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768018.610547:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768018.610552:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:1.0:1775768018.610552:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e26cf977] to pc [ptlrpcd_rcv+-1] req@ffff99420788b100 x1862022294264064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1775768018.610554:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768018.610559:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768018.610570:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207889880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294263808:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768018.610584:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768018.610588:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768018.610590:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768018.610592:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768018.610593:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768018.610599:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002b800. 00000100:00080000:1.0:1775768018.610599:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:0.0:1775768018.610608:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768018.610609:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775768018.610613:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775768018.610615:0:8:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:0.0:1775768018.610619:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768018.610623:0:8:0:(import.c:484:import_select_connection()) Process entered 00000400:00000010:0.0:1775768018.610625:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000100:00080000:1.0:1775768018.610631:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5271 00000400:00000010:0.0:1775768018.610634:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768018.610639:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294263808, portal 10 00000400:00000200:1.0:1775768018.610642:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768018.610649:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768018.610652:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606124672 : -112966103426944 : ffff994207889880) 00000400:00000200:1.0:1775768018.610653:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775768018.610659:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768018.610668:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207889880 x1862022294263808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768043 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775768018.610669:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768018.610677:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768018.610684:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768018.610692:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775768018.610703:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:0.0:1775768018.610710:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000040:1.0:1775768018.610711:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:1.0:1775768018.610725:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775768018.610726:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:1.0:1775768018.610729:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:1.0:1775768018.610731:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:0.0:1775768018.610738:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000100:00000040:1.0:1775768018.610744:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000010:0.0:1775768018.610745:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000001:1.0:1775768018.610750:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:0.0:1775768018.610752:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294263808, offset 0 00000100:00000040:1.0:1775768018.610756:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768018.610760:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000200:0.0:1775768018.610766:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000020:00000040:1.0:1775768018.610767:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:1.0:1775768018.610775:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000400:00000200:0.0:1775768018.610776:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768018.610784:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768018.610788:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 02000000:00000001:1.0:1775768018.610790:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:0.0:1775768018.610793:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768018.610795:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768018.610803:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768018.610805:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 02000000:00000001:1.0:1775768018.610807:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1775768018.610811:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225234:225234:256:4294967295] 192.168.202.156@tcp LPNI seq info [225230:225230:8:4294967295] 02000000:00000001:1.0:1775768018.610815:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768018.610822:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768018.610825:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775768018.610832:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000400:00000200:0.0:1775768018.610833:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768018.610843:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000008:00000001:1.0:1775768018.610843:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1775768018.610851:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000100:00000001:1.0:1775768018.610852:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1775768018.610858:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000010:1.0:1775768018.610860:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207889f80. 00000800:00000200:0.0:1775768018.610861:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000020:00000040:1.0:1775768018.610868:0:8:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00000800:00000200:0.1:1775768018.610871:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768018.610878:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 02000000:00000001:1.0:1775768018.610879:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768018.610883:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1775768018.610886:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000100:00000001:1.0:1775768018.610888:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000800:00000010:0.0:1775768018.610889:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 02000000:00000001:1.0:1775768018.610892:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:0.0:1775768018.610895:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000001:1.0:1775768018.610896:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768018.610908:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 02000000:00000010:1.0:1775768018.610909:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543d000. 00000400:00000200:0.0:1775768018.610911:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775768018.610916:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:1.0:1775768018.610919:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768018.610920:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768018.610924:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207889880 x1862022294263808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768043 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768018.610934:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207889f80 x1862022294264192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768018.610935:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768018.610941:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207889880 x1862022294263808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768043 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768018.611001:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768018.611003:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768018.611008:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768018.611008:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768018.611016:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768018.611018:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ed2274f4] to pc [ptlrpcd_rcv+-1] req@ffff994207889f80 x1862022294264192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775768018.611024:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000100:00000001:1.0:1775768018.611029:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768018.611035:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000040:1.0:1775768018.611035:0:8:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5281) 00000100:00000001:0.0:1775768018.611043:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299943443 00000100:00000001:0.0:1775768018.611047:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299944765 00000100:00000001:0.0:1775768018.611050:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299943443 00000100:00000001:0.0:1775768018.611053:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299944765 00000100:00000001:0.0:1775768018.611056:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768018.611060:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768018.611062:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768018.611069:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768018/real 1775768018] req@ffff994207889880 x1862022294263808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768018.611079:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768018.611081:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768018.611083:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775768018.611086:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768018.611088:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768018.611090:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768018.611094:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207889880 x1862022294263808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768018.611103:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207889880 x1862022294263808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768018.611108:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768018.611109:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768018.611111:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768018.611112:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768018.611114:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207889880 x1862022294263808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768018.611120:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768018.611122:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207889880 x1862022294263808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768043 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768018.611127:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768018.611128:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768018.611129:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768018.611130:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768018.611131:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768018.611133:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768018.611135:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768018.611137:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768018.611139:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768018.611141:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207889880 x1862022294263808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768043 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768018.611146:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207889880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294263808:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768018.611149:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768018.611151:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207888700 x1862022294263936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768018.611175:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768018.611177:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768018.611179:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768018.611182:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768018.611184:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768018.611187:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207888700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294263936:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768018.611190:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768018.611192:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768018.611193:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768018.611195:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768018.611198:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002bc00. 02000000:00000001:0.0:1775768018.611200:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768018.611203:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768018.611205:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775768018.611208:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768018.611210:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294263936, portal 10 00000100:00000001:0.0:1775768018.611211:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768018.611212:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606120192 : -112966103431424 : ffff994207888700) 00000100:00000040:0.0:1775768018.611218:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207888700 x1862022294263936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768043 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768018.611223:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768018.611225:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768018.611227:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775768018.611228:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294263936, offset 0 00000400:00000200:0.0:1775768018.611231:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768018.611233:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768018.611236:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768018.611238:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768018.611240:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768018.611242:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768018.611243:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768018.611246:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225235:225235:256:4294967295] 192.168.202.156@tcp LPNI seq info [225231:225231:8:4294967295] 00000400:00000200:0.0:1775768018.611249:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768018.611253:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768018.611257:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768018.611259:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775768018.611262:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768018.611264:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775768018.611267:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768018.611268:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1775768018.611271:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768018.611273:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775768018.611275:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768018.611278:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768018.611280:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775768018.611281:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775768018.611282:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768018.611284:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207888700 x1862022294263936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768043 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768018.611290:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768018.611292:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207888700 x1862022294263936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768043 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768018.611297:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768018.611298:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768018.611299:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768018.611301:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:0.0:1775768018.611303:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768018.611305:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299943436 00000100:00000001:0.0:1775768018.611306:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299944765 00000100:00000001:0.0:1775768018.611308:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299943436 00000100:00000001:0.0:1775768018.611309:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299944765 00000100:00000001:0.0:1775768018.611311:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768018.611312:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768018.611314:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768018.611316:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768018/real 1775768018] req@ffff994207888700 x1862022294263936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768018.611325:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768018.611326:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768018.611327:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775768018.611329:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768018.611329:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768018.611331:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768018.611333:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207888700 x1862022294263936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768018.611339:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207888700 x1862022294263936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768018.611343:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768018.611344:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768018.611345:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768018.611346:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768018.611348:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207888700 x1862022294263936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768018.611352:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768018.611354:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207888700 x1862022294263936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768043 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768018.611358:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768018.611360:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768018.611362:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768018.611362:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768018.611364:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768018.611366:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768018.611367:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768018.611369:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768018.611371:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768018.611373:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207888700 x1862022294263936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768043 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768018.611377:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207888700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294263936:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768018.611380:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768018.611382:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768018.611384:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207889880 x1862022294263808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768043 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768018.611388:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768018.611390:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768018.611391:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002b800. 02000000:00000001:0.0:1775768018.611392:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768018.611393:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768018.611394:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:0.0:1775768018.611396:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768018.611397:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543e400. 02000000:00000001:0.0:1775768018.611399:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768018.611401:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768018.611402:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207889880. 00000100:00000001:0.0:1775768018.611403:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768018.611404:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768018.611405:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768018.611407:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207888700 x1862022294263936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768043 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768018.611411:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768018.611412:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768018.611413:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002bc00. 02000000:00000001:0.0:1775768018.611414:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768018.611415:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768018.611415:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:0.0:1775768018.611416:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768018.611417:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543dc00. 02000000:00000001:0.0:1775768018.611418:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768018.611419:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768018.611420:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207888700. 00000100:00000001:0.0:1775768018.611421:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768018.611422:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768018.611423:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768018.611425:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768018.611426:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768018.611427:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775768018.611429:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768018.611430:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768018.611432:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff99420788b100 x1862022294264064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768018.611437:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768018.611438:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768018.611440:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768018.611441:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768018.611442:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768018.611444:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff99420788b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294264064:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768018.611446:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768018.611447:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768018.611448:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768018.611450:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768018.611452:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002bc00. 02000000:00000001:0.0:1775768018.611453:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768018.611455:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768018.611457:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775768018.611459:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768018.611461:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294264064, portal 4 00000100:00000001:0.0:1775768018.611463:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768018.611464:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606130944 : -112966103420672 : ffff99420788b100) 00000100:00000040:0.0:1775768018.611467:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff99420788b100 x1862022294264064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768043 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768018.611472:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768018.611473:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768018.611475:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775768018.611476:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294264064, offset 0 00000400:00000200:0.0:1775768018.611479:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768018.611481:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768018.611483:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768018.611484:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768018.611486:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768018.611488:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768018.611490:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768018.611492:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225236:225236:256:4294967295] 192.168.202.156@tcp LPNI seq info [225232:225232:8:4294967295] 00000400:00000200:0.0:1775768018.611495:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768018.611499:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768018.611502:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768018.611504:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775768018.611506:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768018.611508:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775768018.611511:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768018.611513:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1775768018.611515:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768018.611516:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775768018.611518:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768018.611521:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768018.611523:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775768018.611524:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775768018.611525:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768018.611527:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff99420788b100 x1862022294264064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768043 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768018.611532:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768018.611534:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff99420788b100 x1862022294264064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768043 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768018.611539:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768018.611540:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768018.611541:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768018.611543:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:0.0:1775768018.611545:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768018.611546:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299943459 00000100:00000001:0.0:1775768018.611548:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299944765 00000100:00000001:0.0:1775768018.611549:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299943459 00000100:00000001:0.0:1775768018.611552:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299944765 00000100:00000001:0.0:1775768018.611554:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768018.611555:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768018.611556:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768018.611559:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768018/real 1775768018] req@ffff99420788b100 x1862022294264064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768018.611563:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768018.611564:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768018.611565:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775768018.611566:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768018.611567:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768018.611568:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768018.611570:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff99420788b100 x1862022294264064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768018.611575:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff99420788b100 x1862022294264064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768018.611580:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768018.611581:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768018.611582:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768018.611582:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768018.611584:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff99420788b100 x1862022294264064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768018.611588:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768018.611590:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff99420788b100 x1862022294264064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768043 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768018.611594:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768018.611596:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768018.611597:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768018.611597:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768018.611599:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768018.611602:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768018.611604:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768018.611607:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768018.611608:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768018.611610:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff99420788b100 x1862022294264064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768043 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768018.611616:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff99420788b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294264064:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768018.611618:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768018.611621:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207889f80 x1862022294264192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768018.611625:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768018.611626:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768018.611627:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768018.611628:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768018.611629:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768018.611631:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207889f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294264192:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768018.611633:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768018.611634:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768018.611634:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768018.611635:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768018.611637:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002b800. 02000000:00000001:0.0:1775768018.611638:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768018.611641:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768018.611642:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775768018.611644:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768018.611646:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294264192, portal 4 00000100:00000001:0.0:1775768018.611647:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768018.611648:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606126464 : -112966103425152 : ffff994207889f80) 00000100:00000040:0.0:1775768018.611652:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207889f80 x1862022294264192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768043 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768018.611657:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768018.611658:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768018.611659:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775768018.611661:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294264192, offset 0 00000400:00000200:0.0:1775768018.611663:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768018.611665:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768018.611667:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768018.611668:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768018.611670:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768018.611671:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768018.611673:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768018.611675:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225237:225237:256:4294967295] 192.168.202.156@tcp LPNI seq info [225233:225233:8:4294967295] 00000400:00000200:0.0:1775768018.611677:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768018.611681:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768018.611684:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768018.611686:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775768018.611688:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768018.611690:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775768018.611692:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768018.611694:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1775768018.611697:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768018.611698:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775768018.611700:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768018.611702:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768018.611703:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775768018.611704:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775768018.611706:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768018.611707:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207889f80 x1862022294264192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768043 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768018.611712:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768018.611729:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207889f80 x1862022294264192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768043 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768018.611739:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768018.611740:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768018.611742:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768018.611745:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:0.0:1775768018.611748:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768018.611749:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299943462 00000100:00000001:0.0:1775768018.611751:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299944765 00000100:00000001:0.0:1775768018.611752:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299943462 00000100:00000001:0.0:1775768018.611754:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299944765 00000100:00000001:0.0:1775768018.611755:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768018.611757:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768018.611758:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768018.611760:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768018/real 1775768018] req@ffff994207889f80 x1862022294264192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768018.611765:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768018.611766:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768018.611767:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775768018.611769:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768018.611770:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768018.611771:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768018.611773:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207889f80 x1862022294264192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768018.611778:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207889f80 x1862022294264192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768018.611782:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768018.611783:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768018.611784:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768018.611785:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768018.611787:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207889f80 x1862022294264192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768018.611792:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768018.611794:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207889f80 x1862022294264192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768043 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768018.611799:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768018.611800:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768018.611801:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768018.611802:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768018.611803:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768018.611806:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768018.611807:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768018.611809:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768018.611810:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768018.611812:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207889f80 x1862022294264192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768043 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768018.611817:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207889f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294264192:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768018.611819:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768018.611820:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768018.611822:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff99420788b100 x1862022294264064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768043 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768018.611827:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768018.611828:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768018.611829:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002bc00. 02000000:00000001:0.0:1775768018.611831:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768018.611831:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768018.611832:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:0.0:1775768018.611834:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768018.611835:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543fc00. 02000000:00000001:0.0:1775768018.611837:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768018.611838:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768018.611839:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff99420788b100. 00000100:00000001:0.0:1775768018.611841:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768018.611842:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768018.611843:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768018.611845:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207889f80 x1862022294264192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768043 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768018.611850:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768018.611850:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768018.611851:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002b800. 02000000:00000001:0.0:1775768018.611852:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768018.611853:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768018.611853:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:0.0:1775768018.611854:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768018.611855:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543d000. 02000000:00000001:0.0:1775768018.611856:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768018.611857:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768018.611858:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207889f80. 00000100:00000001:0.0:1775768018.611859:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768018.611860:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768018.611861:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768018.611863:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768018.611864:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768018.611865:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775768018.611866:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768018.673298:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768018.673306:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768018.673313:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768018.673315:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768018.673317:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768018.673318:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768018.673321:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768018.673322:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768018.673325:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768018.673326:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768018.673346:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768018.673349:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768018.673352:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768018.737275:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768018.737282:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768018.737284:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768018.737289:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768018.737293:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768018.737297:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768018.737299:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768018.737301:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768018.737304:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768019.633218:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768019.633228:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768019.633230:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768019.633235:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:2.0:1775768019.633236:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775768019.633243:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768019.633244:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768019.633251:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768019.633251:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768019.633253:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768019.633258:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768019.633263:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768019.633264:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00100000:3.0:1775768019.633264:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:0.0:1775768019.633267:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768019.633271:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768019.633276:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768019.633277:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768019.633277:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768019.633287:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768019.633287:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768019.633298:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768019.633300:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768019.633303:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768019.633307:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768019.633310:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768019.633312:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768019.697276:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768019.697283:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768019.697290:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768019.697292:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768019.697296:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768019.697298:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768019.697302:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768019.697303:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768019.697307:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768019.697308:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768019.697334:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768019.697337:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768019.697341:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768019.761308:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768019.761317:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768019.761320:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768019.761328:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768019.761332:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768019.761338:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768019.761342:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768019.761345:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768019.761349:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768019.953262:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299944765, at 4299944768 00000100:00000040:0.0:1775768019.953273:0:14953:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4299944765, at 4299944768 00000100:00000010:1.0:1775768019.953278:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3b40. 00000100:00000010:0.0:1775768019.953283:0:14953:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3d80. 00000100:00000001:3.0:1775768020.657217:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768020.657225:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768020.657225:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768020.657228:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768020.657229:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768020.657231:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768020.657234:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775768020.657235:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775768020.657240:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768020.657240:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768020.657244:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768020.657244:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768020.657249:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768020.657252:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768020.657258:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768020.657259:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768020.657264:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768020.657265:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768020.657270:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768020.657271:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768020.657273:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768020.657280:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768020.657285:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768020.657288:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768020.657291:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768020.657293:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768020.657296:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768020.721292:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768020.721297:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768020.721305:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768020.721307:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768020.721311:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768020.721312:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768020.721314:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768020.721315:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768020.721317:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768020.721318:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768020.721340:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768020.721343:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768020.721345:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768020.785303:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768020.785326:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768020.785328:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768020.785336:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768020.785341:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768020.785347:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768020.785351:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768020.785355:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768020.785359:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768021.681232:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768021.681249:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768021.681257:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768021.681266:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768021.681272:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768021.681281:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768021.681286:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768021.681290:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768021.681294:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768021.681296:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768021.681305:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768021.681307:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768021.681312:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775768021.681319:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768021.681320:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768021.681324:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768021.681326:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768021.681329:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768021.681329:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768021.681335:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768021.681336:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768021.681348:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768021.681349:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768021.681354:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768021.681359:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768021.681361:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768021.681365:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768021.745263:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768021.745271:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768021.745274:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768021.745277:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768021.745282:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768021.745285:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768021.745287:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768021.745288:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768021.745291:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768021.745293:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768021.745296:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768021.745297:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768021.745300:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768021.745301:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768021.745303:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768021.745305:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768021.745326:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768021.745329:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768021.745331:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768021.809283:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768021.809290:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768021.809292:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768021.809298:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768021.809301:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768021.809305:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768021.809307:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768021.809309:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768021.809312:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768022.705294:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768022.705295:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768022.705295:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768022.705306:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768022.705308:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768022.705308:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768022.705318:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768022.705319:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768022.705327:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768022.705330:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:2.0:1775768022.705330:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775768022.705331:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768022.705345:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768022.705345:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768022.705345:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768022.705358:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768022.705358:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768022.705359:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768022.705370:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768022.705373:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768022.705376:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768022.705378:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768022.705381:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768022.705384:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768022.705385:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768022.705388:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768022.705393:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768022.769299:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768022.769305:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768022.769311:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768022.769312:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768022.769316:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768022.769317:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768022.769321:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768022.769322:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768022.769325:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768022.769326:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768022.769346:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768022.769348:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768022.769351:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768022.833259:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768022.833271:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768022.833272:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768022.833279:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768022.833282:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768022.833287:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768022.833289:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768022.833291:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768022.833293:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768023.729243:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768023.729244:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768023.729252:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768023.729253:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768023.729255:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768023.729256:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768023.729263:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775768023.729264:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768023.729269:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768023.729270:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1775768023.729273:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775768023.729274:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768023.729275:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768023.729281:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:2.0:1775768023.729283:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775768023.729284:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768023.729286:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775768023.729288:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768023.729289:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:2.0:1775768023.729290:0:8:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775768023.729293:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768023.729296:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:3.0:1775768023.729297:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768023.729304:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5276 00000080:00000001:0.0:1775768023.729308:0:13471:0:(rw.c:1481:ll_writepages()) Process entered 00000080:00000001:0.0:1775768023.729312:0:13471:0:(file.c:5250:cl_sync_file_range()) Process entered 00000400:00000200:2.0:1775768023.729316:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:0.0:1775768023.729317:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00000001:1.0:1775768023.729322:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000400:00000200:2.0:1775768023.729325:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:1.0:1775768023.729326:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000020:00000010:0.0:1775768023.729329:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff9942103bed80. 00000100:00000001:1.0:1775768023.729330:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768023.729330:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775768023.729335:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000040:2.0:1775768023.729337:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000010:0.0:1775768023.729339:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0f200. 00000100:00000001:1.0:1775768023.729339:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768023.729341:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775768023.729343:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:0.0:1775768023.729346:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210b0f600. 00000100:00000040:2.0:1775768023.729346:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768023.729347:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768023.729349:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768023.729353:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768023.729353:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000010:0.0:1775768023.729354:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942078a3d80. 00000020:00000001:2.0:1775768023.729360:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775768023.729362:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000008:00000010:0.0:1775768023.729364:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff9942086e6468. 00000020:00000001:2.0:1775768023.729365:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775768023.729369:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00020000:00000010:0.0:1775768023.729371:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994209592f00. 00000020:00000001:2.0:1775768023.729372:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:2.0:1775768023.729375:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00008000:00000010:0.0:1775768023.729378:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff99422035b880. 00000100:00000040:2.0:1775768023.729378:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768023.729380:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768023.729383:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768023.729385:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000080:00000010:0.0:1775768023.729386:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994207e3e150. 00000080:00000010:0.0:1775768023.729391:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b2238. 00000020:00000010:0.0:1775768023.729395:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0fa00. 00000020:00000040:2.0:1775768023.729395:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000008:00000010:0.0:1775768023.729401:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff99420906be40. 00000100:00080000:2.0:1775768023.729405:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00020000:00000010:0.0:1775768023.729406:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff9942103ccfc0. 00000100:00000001:2.0:1775768023.729407:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:0.0:1775768023.729411:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076afa0. 02000000:00000001:2.0:1775768023.729411:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000080:00000010:0.0:1775768023.729416:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942107690f0. 02000000:00000001:2.0:1775768023.729417:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768023.729421:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768023.729422:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107752095112 : -112965957456504 : ffff9942103bed88) 00000100:00000001:2.0:1775768023.729423:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000020:00001000:0.0:1775768023.729427:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff9942103bed80 00000100:00000001:2.0:1775768023.729427:0:8:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768023.729432:0:13471:0:(cl_object.c:151:cl_object_top()) ffff99420791a5f8 -> ffff99420791a5f8 00000020:00000001:0.0:1775768023.729435:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000100:00000010:2.0:1775768023.729435:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfdc00. 00000080:00000001:0.0:1775768023.729437:0:13471:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:0.0:1775768023.729441:0:13471:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x10176:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000040:2.0:1775768023.729441:0:8:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 00000020:00000001:0.0:1775768023.729448:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 02000000:00000001:2.0:1775768023.729449:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000020:00000001:0.0:1775768023.729450:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 02000000:00000001:2.0:1775768023.729451:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1775768023.729453:0:13471:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768023.729456:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00020000:00000002:0.0:1775768023.729458:0:13471:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x10176:0x0]io ffff9942077b2298 type 6 ignore/verify layout 1/0 02000000:00000001:2.0:1775768023.729460:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00020000:00000002:0.0:1775768023.729461:0:13471:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207dfb6c8) owner 0000000000000000/ffff994220080000 02000000:00000001:2.0:1775768023.729465:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768023.729466:0:13471:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1775768023.729469:0:13471:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1775768023.729472:0:13471:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00020000:00000001:0.0:1775768023.729473:0:13471:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768023.729474:0:13471:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1775768023.729476:0:13471:0:(lov_io.c:203:lov_io_subio_init()) Process entered 02000000:00000010:2.0:1775768023.729477:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1d400. 00020000:00000001:0.0:1775768023.729478:0:13471:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768023.729481:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1775768023.729483:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1775768023.729484:0:13471:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00020000:00000002:0.0:1775768023.729485:0:13471:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207dfb6c8) owner 0000000000000000/ffff994220080000 00000020:00000001:0.0:1775768023.729487:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768023.729489:0:13471:0:(cl_io.c:823:cl_io_loop()) Process entered 00000100:00000001:2.0:1775768023.729490:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768023.729491:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00020000:00000001:0.0:1775768023.729494:0:13471:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00020000:00200000:0.0:1775768023.729498:0:13471:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00000002:0.0:1775768023.729501:0:13471:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:0.0:1775768023.729504:0:13471:0:(lov_io.c:155:lov_sub_get()) Process entered 00000100:00100000:2.0:1775768023.729506:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfdc00 x1862022294264320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00020000:00000001:0.0:1775768023.729507:0:13471:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000020:00000001:0.0:1775768023.729509:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:0.0:1775768023.729513:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff9942103be000. 00000020:00000010:0.0:1775768023.729515:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0ec00. 00000020:00000010:0.0:1775768023.729517:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210b0ea00. 00000020:00000010:0.0:1775768023.729519:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942078a3c00. 00000008:00000010:0.0:1775768023.729521:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff9942086e4e18. 00020000:00000010:0.0:1775768023.729523:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994209592e40. 00008000:00000010:0.0:1775768023.729528:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994220358000. 00000080:00000010:0.0:1775768023.729533:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994207e3e3f0. 00000080:00000010:0.0:1775768023.729536:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b0ff8. 00000100:00000001:2.0:1775768023.729538:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000020:00000010:0.0:1775768023.729539:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0fc00. 00000008:00000010:0.0:1775768023.729543:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff99420906a6e0. 00020000:00000010:0.0:1775768023.729546:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff9942103cc480. 00000100:00000001:2.0:1775768023.729547:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:0.0:1775768023.729549:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076adb0. 00000080:00000010:0.0:1775768023.729553:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff994210769870. 00000020:00000001:0.0:1775768023.729556:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107752091656 : -112965957459960 : ffff9942103be008) 00000020:00001000:0.0:1775768023.729558:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff9942103be000 00000100:00000040:2.0:1775768023.729558:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003e25603c] to pc [ptlrpcd_rcv+-1] req@ffff994207dfdc00 x1862022294264320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775768023.729561:0:13471:0:(cl_object.c:151:cl_object_top()) ffff99420791ae90 -> ffff99420791a5f8 00000020:00000001:0.0:1775768023.729562:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:0.0:1775768023.729566:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1775768023.729567:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:0.0:1775768023.729570:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768023.729574:0:13471:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768023.729577:0:13471:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107752153128 : -112965957398488 : ffff9942103cd028) 00000020:00000001:0.0:1775768023.729581:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000020:00000001:0.0:1775768023.729583:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768023.729585:0:13471:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768023.729586:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768023.729588:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00020000:00000001:0.0:1775768023.729589:0:13471:0:(lov_io.c:1108:lov_io_lock()) Process entered 00020000:00000001:0.0:1775768023.729591:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:0.0:1775768023.729591:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:0.0:1775768023.729592:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:0.0:1775768023.729594:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768023.729595:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768023.729596:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768023.729601:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768023.729602:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00020000:00000001:0.0:1775768023.729603:0:13471:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768023.729608:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000100:00080000:2.0:1775768023.729609:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000020:00000001:0.0:1775768023.729610:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768023.729613:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768023.729613:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000020:00000001:0.0:1775768023.729616:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00000100:00080000:2.0:1775768023.729616:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00020000:00000001:0.0:1775768023.729620:0:13471:0:(lov_io.c:1114:lov_io_start()) Process entered 00000100:00000001:2.0:1775768023.729621:0:8:0:(import.c:484:import_select_connection()) Process entered 00020000:00000001:0.0:1775768023.729623:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:0.0:1775768023.729625:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00000100:00000001:1.0:1775768023.729626:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:2.0:1775768023.729626:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5276 00000008:00000001:0.0:1775768023.729629:0:13471:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000100:00000001:1.0:1775768023.729631:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000008:00000001:0.0:1775768023.729632:0:13471:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000400:00000200:2.0:1775768023.729634:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768023.729635:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000400:00000200:2.0:1775768023.729640:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000008:00000001:0.0:1775768023.729642:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000100:00080000:2.0:1775768023.729643:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775768023.729644:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfdc00 x1862022294264320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000020:0.0:1775768023.729645:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:0.0:1775768023.729649:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768023.729650:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000008:00000001:0.0:1775768023.729656:0:13471:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000100:00000001:2.0:1775768023.729656:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000008:00000001:0.0:1775768023.729660:0:13471:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000100:00000040:2.0:1775768023.729662:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000008:00000001:0.0:1775768023.729675:0:13471:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107754940544 : -112965954611072 : ffff994210675880) 00000100:00000001:1.0:1775768023.729677:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768023.729677:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:1.0:1775768023.729680:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768023.729681:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000008:00000020:0.0:1775768023.729682:0:13471:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994210675880 ready 0|-|- wr 1|-|+ rd 0|- 10 in flight 00000020:00000040:2.0:1775768023.729684:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 02000000:00000001:1.0:1775768023.729687:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:2.0:1775768023.729688:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000008:00000001:0.0:1775768023.729689:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 02000000:00000001:1.0:1775768023.729691:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000020:0.0:1775768023.729692:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000020:00000040:2.0:1775768023.729692:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000008:00000001:0.0:1775768023.729695:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1775768023.729697:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000008:00000001:0.0:1775768023.729699:0:13471:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 02000000:00000001:1.0:1775768023.729699:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768023.729701:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000008:00000020:0.0:1775768023.729704:0:13471:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994218232fc0 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000100:00100000:1.0:1775768023.729706:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294264320:192.168.202.156@tcp:250:kworker.0 00000100:00000040:2.0:1775768023.729708:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768023.729714:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000008:00000001:0.0:1775768023.729715:0:13471:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768023.729717:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000008:00000001:0.0:1775768023.729719:0:13471:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768023.729719:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000001:1.0:1775768023.729722:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:0.0:1775768023.729725:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768023.729725:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 02000000:00000001:1.0:1775768023.729726:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768023.729730:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775768023.729732:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00020000:00000001:0.0:1775768023.729733:0:13471:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768023.729733:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:2.0:1775768023.729736:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000020:00000001:0.0:1775768023.729738:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768023.729741:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768023.729742:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 02000000:00000010:1.0:1775768023.729744:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543d400. 00020000:00000001:0.0:1775768023.729745:0:13471:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 02000000:00000001:2.0:1775768023.729745:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00020000:00000001:0.0:1775768023.729749:0:13471:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 02000000:00000001:1.0:1775768023.729749:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768023.729751:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 02000000:00000001:2.0:1775768023.729754:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1775768023.729755:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00020000:00000001:0.0:1775768023.729757:0:13471:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768023.729758:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768023.729759:0:13471:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000020:00000001:0.0:1775768023.729762:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00000400:00000200:1.0:1775768023.729764:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768023.729764:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000020:00000001:0.0:1775768023.729765:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00020000:00000001:0.0:1775768023.729768:0:13471:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00020000:00000001:0.0:1775768023.729770:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:0.0:1775768023.729771:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000008:00100000:2.0:1775768023.729771:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000400:00000010:1.0:1775768023.729773:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023cc08. 00000020:00000001:0.0:1775768023.729774:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000008:00000001:2.0:1775768023.729775:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768023.729776:0:13471:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768023.729780:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768023.729780:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768023.729784:0:13471:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000020:00000001:0.0:1775768023.729785:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:0.0:1775768023.729787:0:13471:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000400:00000010:1.0:1775768023.729787:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf67f8. 00000100:00000010:2.0:1775768023.729787:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dff100. 00000020:00000001:0.0:1775768023.729791:0:13471:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000040:2.0:1775768023.729792:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:0.0:1775768023.729793:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:0.0:1775768023.729797:0:13471:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 02000000:00000001:2.0:1775768023.729797:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00020000:00000001:0.0:1775768023.729798:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:0.0:1775768023.729800:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 02000000:00000001:2.0:1775768023.729800:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775768023.729801:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294264320, portal 25 00000020:00000001:0.0:1775768023.729803:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000100:00000001:2.0:1775768023.729805:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00020000:00000001:0.0:1775768023.729807:0:13471:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768023.729808:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1775768023.729808:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00020000:00000001:0.0:1775768023.729810:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768023.729811:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611843584 : -112966097708032 : ffff994207dfdc00) 02000000:00000001:2.0:1775768023.729812:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768023.729813:0:13471:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000020:00000001:0.0:1775768023.729815:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000020:00000001:0.0:1775768023.729818:0:13471:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768023.729820:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 02000000:00000010:2.0:1775768023.729820:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1c800. 00020000:00000001:0.0:1775768023.729824:0:13471:0:(lov_io.c:675:lov_io_fini()) Process entered 00000100:00000040:1.0:1775768023.729824:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfdc00 x1862022294264320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768048 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768023.729826:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768023.729827:0:13471:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:0.0:1775768023.729830:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:0.0:1775768023.729833:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1775768023.729836:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff9942103be000 00000100:00100000:2.0:1775768023.729837:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dff100 x1862022294264448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775768023.729842:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000010:0.0:1775768023.729843:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210b0ea00. 00000100:00000040:1.0:1775768023.729850:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000020:00000010:0.0:1775768023.729852:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942078a3c00. 00000400:00000010:1.0:1775768023.729856:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000100:00000001:2.0:1775768023.729857:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000008:00000010:0.0:1775768023.729859:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff9942086e4e18. 00000100:00000001:2.0:1775768023.729862:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775768023.729863:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294264320, offset 0 00020000:00000010:0.0:1775768023.729865:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994209592e40. 00000100:00000040:2.0:1775768023.729868:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3e64ae1] to pc [ptlrpcd_rcv+-1] req@ffff994207dff100 x1862022294264448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00008000:00000010:0.0:1775768023.729870:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994220358000. 00000400:00000200:1.0:1775768023.729876:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000080:00000010:0.0:1775768023.729877:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994207e3e3f0. 00000080:00000010:0.0:1775768023.729884:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b0ff8. 00000100:00000001:2.0:1775768023.729884:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768023.729890:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00080000:2.0:1775768023.729890:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000010:0.0:1775768023.729891:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0ec00. 00000008:00000010:0.0:1775768023.729897:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff99420906a6e0. 00000100:00080000:2.0:1775768023.729898:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775768023.729901:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:1.0:1775768023.729902:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00080000:2.0:1775768023.729903:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00020000:00000010:0.0:1775768023.729904:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff9942103cc480. 00008000:00000010:0.0:1775768023.729910:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076adb0. 00000100:00000001:2.0:1775768023.729910:0:8:0:(import.c:484:import_select_connection()) Process entered 00000080:00000010:0.0:1775768023.729914:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff994210769870. 00000100:00080000:2.0:1775768023.729914:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5276 00000400:00000200:1.0:1775768023.729918:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000010:0.0:1775768023.729919:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0fc00. 00000400:00000200:2.0:1775768023.729922:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000010:0.0:1775768023.729924:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff9942103be000. 00000400:00000200:1.0:1775768023.729926:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00020000:00000001:0.0:1775768023.729928:0:13471:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00000400:00000200:2.0:1775768023.729929:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00020000:00000001:0.0:1775768023.729931:0:13471:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000100:00080000:2.0:1775768023.729932:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000080:00000001:0.0:1775768023.729934:0:13471:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000400:00000200:1.0:1775768023.729935:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000100:00000040:2.0:1775768023.729938:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000080:00200000:0.0:1775768023.729939:0:13471:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x10176:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000400:00000200:1.0:1775768023.729942:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00000001:2.0:1775768023.729945:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000080:00000001:0.0:1775768023.729947:0:13471:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1775768023.729950:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000100:00000040:2.0:1775768023.729951:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00001000:0.0:1775768023.729953:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff9942103bed80 00000400:00000200:1.0:1775768023.729954:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225238:225238:256:4294967295] 192.168.202.156@tcp LPNI seq info [225234:225234:8:4294967295] 00000100:00000001:2.0:1775768023.729955:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000010:0.0:1775768023.729957:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210b0f600. 00000020:00000001:2.0:1775768023.729961:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000010:0.0:1775768023.729963:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942078a3d80. 00000020:00000040:2.0:1775768023.729964:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000008:00000010:0.0:1775768023.729966:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff9942086e6468. 00000020:00000001:2.0:1775768023.729968:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:1.0:1775768023.729969:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000010:0.0:1775768023.729970:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994209592f00. 00000020:00000040:2.0:1775768023.729972:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00008000:00000010:0.0:1775768023.729975:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff99422035b880. 00000020:00000001:2.0:1775768023.729975:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000080:00000010:0.0:1775768023.729980:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994207e3e150. 00000020:00000001:2.0:1775768023.729981:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000080:00000010:0.0:1775768023.729986:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b2238. 00000100:00000040:2.0:1775768023.729989:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000200:1.0:1775768023.729991:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000020:00000010:0.0:1775768023.729992:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0f200. 00000100:00000001:2.0:1775768023.729992:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000008:00000010:0.0:1775768023.729998:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff99420906be40. 00000100:00000040:2.0:1775768023.729998:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768023.730003:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00020000:00000010:0.0:1775768023.730004:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff9942103ccfc0. 00000800:00000200:1.0:1775768023.730004:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000020:00000040:2.0:1775768023.730008:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00008000:00000010:0.0:1775768023.730010:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076afa0. 00000800:00000010:1.0:1775768023.730012:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541200. 00000100:00080000:2.0:1775768023.730013:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000080:00000010:0.0:1775768023.730017:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942107690f0. 00000100:00000001:2.0:1775768023.730018:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1775768023.730022:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0fa00. 02000000:00000001:2.0:1775768023.730022:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:1.0:1775768023.730025:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000010:0.0:1775768023.730026:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff9942103bed80. 02000000:00000001:2.0:1775768023.730027:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000001:0.0:1775768023.730032:0:13471:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1775768023.730032:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775768023.730033:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000080:00000001:0.0:1775768023.730038:0:13471:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768023.730038:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775768023.730041:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000200:1.1:1775768023.730044:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000008:00000001:2.0:1775768023.730045:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768023.730049:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775768023.730051:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000100:00000010:2.0:1775768023.730053:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfc700. 00000800:00000100:1.0:1775768023.730058:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000020:00000040:2.0:1775768023.730058:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000800:00000010:1.0:1775768023.730063:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541200. 02000000:00000001:2.0:1775768023.730063:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775768023.730065:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768023.730068:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:1.0:1775768023.730069:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000001:2.0:1775768023.730070:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775768023.730073:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768023.730077:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 02000000:00000010:2.0:1775768023.730079:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1ec00. 00000400:00000200:1.0:1775768023.730082:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000100:00000001:2.0:1775768023.730082:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775768023.730086:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000100:00100000:2.0:1775768023.730089:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfc700 x1862022294264576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775768023.730091:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1775768023.730098:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000200:1.0:1775768023.730099:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfdc00 x1862022294264320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768048 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768023.730102:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768023.730109:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a176dd1] to pc [ptlrpcd_rcv+-1] req@ffff994207dfc700 x1862022294264576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768023.730112:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768023.730118:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfdc00 x1862022294264320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768048 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768023.730118:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768023.730123:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768023.730130:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1775768023.730132:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775768023.730135:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768023.730135:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768023.730138:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768023.730138:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775768023.730142:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5276 00000100:00000001:1.0:1775768023.730145:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768023.730149:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768023.730153:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000400:00000200:1.0:1775768023.730175:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768023.730179:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299949886 00000100:00080000:2.0:1775768023.730184:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000010:1.0:1775768023.730186:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba9c0. 00000100:00000040:2.0:1775768023.730187:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768023.730191:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768023.730193:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768023.730195:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:1.0:1775768023.730196:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000020:00000001:2.0:1775768023.730197:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775768023.730200:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:2.0:1775768023.730202:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:1.0:1775768023.730204:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299949886 00000020:00000040:2.0:1775768023.730204:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:2.0:1775768023.730205:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000010:1.0:1775768023.730210:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba120. 00000020:00000001:2.0:1775768023.730210:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:1.0:1775768023.730217:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000040:2.0:1775768023.730217:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768023.730222:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775768023.730223:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1775768023.730223:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768023.730229:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768023.730229:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775768023.730231:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00000001:1.0:1775768023.730233:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00080000:2.0:1775768023.730236:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775768023.730239:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768023.730240:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768023/real 1775768023] req@ffff994207dfdc00 x1862022294264320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1775768023.730241:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775768023.730245:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768023.730250:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768023.730254:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775768023.730257:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000400:00000001:1.0:1775768023.730259:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000008:00000001:2.0:1775768023.730260:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1775768023.730261:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1775768023.730263:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775768023.730265:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023cc08. 00000100:00000010:2.0:1775768023.730267:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfe300. 00000400:00000200:1.0:1775768023.730269:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf67f8 00000020:00000040:2.0:1775768023.730270:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00000400:00000010:1.0:1775768023.730273:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf67f8. 02000000:00000001:2.0:1775768023.730274:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775768023.730277:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768023.730278:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1775768023.730279:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775768023.730281:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000200:1.0:1775768023.730282:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfdc00 x1862022294264320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1775768023.730283:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775768023.730288:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1fc00. 00000100:00000001:2.0:1775768023.730293:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775768023.730297:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfdc00 x1862022294264320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768023.730301:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfe300 x1862022294264704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775768023.730317:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768023.730318:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775768023.730320:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768023.730323:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768023.730324:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768023.730327:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768023.730328:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c661fef] to pc [ptlrpcd_rcv+-1] req@ffff994207dfe300 x1862022294264704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768023.730334:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfdc00 x1862022294264320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768023.730340:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768023.730345:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775768023.730348:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1775768023.730349:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1775768023.730352:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775768023.730354:0:8:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:1.0:1775768023.730357:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfdc00 x1862022294264320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768048 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768023.730357:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775768023.730362:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5276 00000400:00000200:2.0:1775768023.730369:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768023.730374:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:1.0:1775768023.730375:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768023.730376:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1775768023.730381:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768023.730382:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768023.730386:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768023.730387:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775768023.730389:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768023.730390:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768023.730394:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768023.730394:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775768023.730396:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:1.0:1775768023.730397:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000040:2.0:1775768023.730398:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:2.0:1775768023.730401:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00080000:1.0:1775768023.730403:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000020:00000040:2.0:1775768023.730403:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:2.0:1775768023.730408:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00080000:1.0:1775768023.730411:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000020:00000001:2.0:1775768023.730411:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000001:1.0:1775768023.730416:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768023.730417:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768023.730420:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768023.730423:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfdc00 x1862022294264320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768048 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775768023.730424:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768023.730429:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775768023.730434:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:2.0:1775768023.730439:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00100000:1.0:1775768023.730442:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294264320:192.168.202.156@tcp:250:kworker.0 00000100:00000001:2.0:1775768023.730443:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768023.730446:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1775768023.730450:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1775768023.730450:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768023.730453:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768023.730454:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1775768023.730456:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775768023.730459:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000100:00000040:1.0:1775768023.730460:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfdc00 x1862022294264320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768048 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:2.0:1775768023.730464:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768023.730468:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775768023.730473:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfed80. 00000100:00000001:1.0:1775768023.730477:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 00000020:00000040:2.0:1775768023.730477:0:8:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:1.0:1775768023.730482:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1775768023.730482:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775768023.730483:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768023.730485:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000010:1.0:1775768023.730486:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543d400. 02000000:00000001:2.0:1775768023.730486:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775768023.730489:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768023.730490:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768023.730492:0:2399:0:(genops.c:1236:class_import_put()) Process entered 02000000:00000010:2.0:1775768023.730493:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1f000. 00000020:00000040:1.0:1775768023.730495:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000100:00000001:2.0:1775768023.730497:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768023.730499:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768023.730502:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1d400. 00000100:00100000:2.0:1775768023.730502:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfed80 x1862022294264832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1775768023.730507:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:2.0:1775768023.730509:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1775768023.730510:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1775768023.730512:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775768023.730513:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfdc00. 00000100:00000001:1.0:1775768023.730518:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000040:2.0:1775768023.730518:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb9b9eae] to pc [ptlrpcd_rcv+-1] req@ffff994207dfed80 x1862022294264832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768023.730521:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768023.730525:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768023.730532:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768023.730533:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768023.730536:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768023.730536:0:8:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5286) 00000100:00000001:1.0:1775768023.730541:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768023.730543:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768023.730544:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768023.730548:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dff100 x1862022294264448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768023.730554:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768023.730555:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768023.730557:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768023.730559:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768023.730560:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768023.730562:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294264448:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768023.730564:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768023.730566:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768023.730567:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768023.730568:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768023.730571:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543d400. 02000000:00000001:1.0:1775768023.730572:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768023.730575:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768023.730577:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023cc08. 00000400:00000010:1.0:1775768023.730580:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf67f8. 00000100:00000200:1.0:1775768023.730582:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294264448, portal 10 00000100:00000001:1.0:1775768023.730584:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768023.730585:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611848960 : -112966097702656 : ffff994207dff100) 00000100:00000040:1.0:1775768023.730590:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dff100 x1862022294264448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768048 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768023.730595:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768023.730597:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768023.730598:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000100:00000200:1.0:1775768023.730601:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294264448, offset 0 00000400:00000200:1.0:1775768023.730605:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768023.730609:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768023.730613:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768023.730616:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768023.730619:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768023.730622:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768023.730624:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768023.730628:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225239:225239:256:4294967295] 192.168.202.156@tcp LPNI seq info [225235:225235:8:4294967295] 00000400:00000200:1.0:1775768023.730633:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768023.730640:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768023.730645:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768023.730649:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541200. 00000800:00000200:1.0:1775768023.730654:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768023.730657:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775768023.730662:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768023.730682:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000100:1.0:1775768023.730688:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768023.730690:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541200. 00000400:00000200:1.0:1775768023.730694:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768023.730698:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768023.730701:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000400:00000010:1.0:1775768023.730703:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000100:00000001:1.0:1775768023.730706:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768023.730710:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dff100 x1862022294264448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768048 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768023.730720:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768023.730723:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dff100 x1862022294264448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768048 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768023.730738:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768023.730740:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768023.730744:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768023.730749:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:1.0:1775768023.730752:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768023.730755:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299948563 00000100:00000001:1.0:1775768023.730758:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299949886 00000100:00000001:1.0:1775768023.730760:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299948563 00000100:00000001:1.0:1775768023.730763:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299949886 00000100:00000001:1.0:1775768023.730765:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768023.730767:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768023.730769:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768023.730774:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768023/real 1775768023] req@ffff994207dff100 x1862022294264448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768023.730784:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768023.730786:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768023.730788:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023cc08. 00000400:00000200:1.0:1775768023.730791:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf67f8 00000400:00000010:1.0:1775768023.730793:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf67f8. 00000100:00000001:1.0:1775768023.730795:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768023.730799:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dff100 x1862022294264448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768023.730809:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dff100 x1862022294264448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768023.730817:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768023.730818:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768023.730821:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768023.730822:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768023.730826:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dff100 x1862022294264448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768023.730835:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768023.730839:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dff100 x1862022294264448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768048 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768023.730848:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768023.730851:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768023.730853:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768023.730854:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768023.730856:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768023.730859:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768023.730862:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768023.730866:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768023.730868:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768023.730873:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dff100 x1862022294264448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768048 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768023.730882:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294264448:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768023.730887:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768023.730891:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfc700 x1862022294264576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768023.730899:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768023.730900:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768023.730903:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768023.730905:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768023.730907:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768023.730911:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294264576:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768023.730914:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768023.730916:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768023.730919:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768023.730921:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768023.730925:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543d000. 02000000:00000001:1.0:1775768023.730927:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768023.730931:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768023.730934:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023cc08. 00000400:00000010:1.0:1775768023.730937:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf67f8. 00000100:00000200:1.0:1775768023.730940:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294264576, portal 10 00000100:00000001:1.0:1775768023.730943:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768023.730944:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611838208 : -112966097713408 : ffff994207dfc700) 00000100:00000040:1.0:1775768023.730950:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfc700 x1862022294264576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768048 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768023.730958:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768023.730961:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768023.730963:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000100:00000200:1.0:1775768023.730966:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294264576, offset 0 00000400:00000200:1.0:1775768023.730970:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768023.730975:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768023.730979:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768023.730982:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768023.730985:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768023.730988:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768023.730991:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768023.730994:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225240:225240:256:4294967295] 192.168.202.156@tcp LPNI seq info [225236:225236:8:4294967295] 00000400:00000200:1.0:1775768023.730999:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768023.731007:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768023.731012:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768023.731016:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541200. 00000800:00000200:1.0:1775768023.731020:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768023.731024:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775768023.731028:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768023.731032:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000100:1.0:1775768023.731036:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768023.731039:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541200. 00000400:00000200:1.0:1775768023.731042:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768023.731046:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768023.731049:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000400:00000010:1.0:1775768023.731051:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000100:00000001:1.0:1775768023.731054:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768023.731057:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfc700 x1862022294264576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768048 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768023.731067:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768023.731070:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfc700 x1862022294264576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768048 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768023.731078:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768023.731080:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768023.731083:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768023.731086:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:1.0:1775768023.731090:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768023.731093:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299948555 00000100:00000001:1.0:1775768023.731095:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299949886 00000100:00000001:1.0:1775768023.731098:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299948555 00000100:00000001:1.0:1775768023.731101:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299949886 00000100:00000001:1.0:1775768023.731103:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768023.731105:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768023.731108:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768023.731111:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768023/real 1775768023] req@ffff994207dfc700 x1862022294264576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768023.731120:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768023.731122:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768023.731123:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023cc08. 00000400:00000200:1.0:1775768023.731126:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf67f8 00000400:00000010:1.0:1775768023.731128:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf67f8. 00000100:00000001:1.0:1775768023.731131:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768023.731134:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfc700 x1862022294264576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768023.731144:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfc700 x1862022294264576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768023.731152:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768023.731153:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768023.731177:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768023.731178:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768023.731183:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfc700 x1862022294264576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768023.731192:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768023.731196:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfc700 x1862022294264576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768048 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768023.731204:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768023.731207:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768023.731209:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768023.731211:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768023.731213:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768023.731216:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768023.731219:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768023.731223:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768023.731225:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768023.731229:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfc700 x1862022294264576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768048 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768023.731238:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294264576:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768023.731243:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768023.731247:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfe300 x1862022294264704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768023.731255:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768023.731256:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768023.731259:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768023.731261:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768023.731263:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768023.731266:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294264704:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768023.731270:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768023.731272:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768023.731274:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768023.731276:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768023.731280:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543fc00. 02000000:00000001:1.0:1775768023.731283:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768023.731287:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768023.731290:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023cc08. 00000400:00000010:1.0:1775768023.731294:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf67f8. 00000100:00000200:1.0:1775768023.731297:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294264704, portal 4 00000100:00000001:1.0:1775768023.731299:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768023.731300:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611845376 : -112966097706240 : ffff994207dfe300) 00000100:00000040:1.0:1775768023.731306:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfe300 x1862022294264704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768048 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768023.731314:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768023.731317:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768023.731320:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000100:00000200:1.0:1775768023.731322:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294264704, offset 0 00000400:00000200:1.0:1775768023.731327:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768023.731332:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768023.731336:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768023.731339:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768023.731342:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768023.731345:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768023.731348:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768023.731352:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225241:225241:256:4294967295] 192.168.202.156@tcp LPNI seq info [225237:225237:8:4294967295] 00000400:00000200:1.0:1775768023.731356:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768023.731363:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768023.731368:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768023.731372:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541200. 00000800:00000200:1.0:1775768023.731376:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768023.731380:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775768023.731385:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768023.731388:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000100:1.0:1775768023.731392:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768023.731394:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541200. 00000400:00000200:1.0:1775768023.731398:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768023.731402:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768023.731406:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000400:00000010:1.0:1775768023.731408:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000100:00000001:1.0:1775768023.731410:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768023.731414:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfe300 x1862022294264704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768048 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768023.731423:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768023.731426:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfe300 x1862022294264704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768048 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768023.731434:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768023.731436:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768023.731439:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768023.731443:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:1.0:1775768023.731447:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768023.731449:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299948579 00000100:00000001:1.0:1775768023.731452:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299949886 00000100:00000001:1.0:1775768023.731454:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299948579 00000100:00000001:1.0:1775768023.731457:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299949886 00000100:00000001:1.0:1775768023.731459:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768023.731462:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768023.731464:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768023.731468:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768023/real 1775768023] req@ffff994207dfe300 x1862022294264704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768023.731477:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768023.731479:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768023.731481:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023cc08. 00000400:00000200:1.0:1775768023.731483:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf67f8 00000400:00000010:1.0:1775768023.731485:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf67f8. 00000100:00000001:1.0:1775768023.731488:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768023.731491:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfe300 x1862022294264704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768023.731501:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfe300 x1862022294264704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768023.731508:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768023.731510:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768023.731512:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768023.731514:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768023.731518:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfe300 x1862022294264704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768023.731526:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768023.731530:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfe300 x1862022294264704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768048 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768023.731538:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768023.731541:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768023.731543:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768023.731544:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768023.731547:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768023.731551:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768023.731554:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768023.731558:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768023.731560:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768023.731565:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfe300 x1862022294264704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768048 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768023.731574:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294264704:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768023.731579:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768023.731582:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfed80 x1862022294264832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768023.731590:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768023.731591:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768023.731594:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768023.731596:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768023.731598:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768023.731601:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294264832:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768023.731605:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768023.731607:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768023.731609:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768023.731612:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768023.731615:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543dc00. 02000000:00000001:1.0:1775768023.731617:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768023.731621:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768023.731624:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023cc08. 00000400:00000010:1.0:1775768023.731627:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf67f8. 00000100:00000200:1.0:1775768023.731630:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294264832, portal 4 00000100:00000001:1.0:1775768023.731633:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768023.731634:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611848064 : -112966097703552 : ffff994207dfed80) 00000100:00000040:1.0:1775768023.731641:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfed80 x1862022294264832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768048 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768023.731649:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768023.731652:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768023.731654:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000100:00000200:1.0:1775768023.731657:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294264832, offset 0 00000400:00000200:1.0:1775768023.731661:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768023.731680:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768023.731684:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768023.731687:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768023.731690:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768023.731694:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768023.731696:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768023.731700:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225242:225242:256:4294967295] 192.168.202.156@tcp LPNI seq info [225238:225238:8:4294967295] 00000400:00000200:1.0:1775768023.731705:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768023.731712:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768023.731717:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768023.731722:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541200. 00000800:00000200:1.0:1775768023.731726:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768023.731730:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775768023.731734:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768023.731738:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000100:1.0:1775768023.731742:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768023.731744:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541200. 00000400:00000200:1.0:1775768023.731748:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768023.731751:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768023.731754:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000400:00000010:1.0:1775768023.731756:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000100:00000001:1.0:1775768023.731759:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768023.731763:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfed80 x1862022294264832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768048 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768023.731772:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768023.731775:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfed80 x1862022294264832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768048 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768023.731783:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768023.731785:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768023.731788:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768023.731792:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:1.0:1775768023.731796:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768023.731798:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299948582 00000100:00000001:1.0:1775768023.731801:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299949886 00000100:00000001:1.0:1775768023.731803:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299948582 00000100:00000001:1.0:1775768023.731805:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299949886 00000100:00000001:1.0:1775768023.731807:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768023.731809:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768023.731811:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768023.731815:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768023/real 1775768023] req@ffff994207dfed80 x1862022294264832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768023.731823:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768023.731825:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768023.731827:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023cc08. 00000400:00000200:1.0:1775768023.731829:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf67f8 00000400:00000010:1.0:1775768023.731831:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf67f8. 00000100:00000001:1.0:1775768023.731834:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768023.731837:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfed80 x1862022294264832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768023.731846:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfed80 x1862022294264832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768023.731854:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768023.731856:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768023.731858:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768023.731859:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768023.731863:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfed80 x1862022294264832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768023.731870:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768023.731874:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfed80 x1862022294264832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768048 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768023.731881:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768023.731883:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768023.731884:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768023.731885:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768023.731886:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768023.731890:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768023.731892:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768023.731894:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768023.731896:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768023.731899:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfed80 x1862022294264832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768048 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768023.731903:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294264832:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768023.731905:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768023.731907:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768023.731908:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dff100 x1862022294264448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768048 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768023.731913:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768023.731914:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768023.731916:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543d400. 02000000:00000001:1.0:1775768023.731918:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768023.731919:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768023.731920:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:1.0:1775768023.731921:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768023.731923:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1c800. 02000000:00000001:1.0:1775768023.731925:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768023.731926:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768023.731927:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dff100. 00000100:00000001:1.0:1775768023.731929:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768023.731930:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768023.731931:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768023.731933:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfc700 x1862022294264576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768048 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768023.731938:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768023.731939:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768023.731939:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543d000. 02000000:00000001:1.0:1775768023.731940:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768023.731941:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768023.731942:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:1.0:1775768023.731943:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768023.731944:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1ec00. 02000000:00000001:1.0:1775768023.731946:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768023.731946:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768023.731947:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfc700. 00000100:00000001:1.0:1775768023.731948:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768023.731949:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768023.731950:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768023.731952:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfe300 x1862022294264704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768048 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768023.731955:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768023.731956:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768023.731957:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543fc00. 02000000:00000001:1.0:1775768023.731958:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768023.731959:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768023.731960:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:1.0:1775768023.731961:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768023.731961:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1fc00. 02000000:00000001:1.0:1775768023.731963:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768023.731963:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768023.731964:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfe300. 00000100:00000001:1.0:1775768023.731966:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768023.731966:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768023.731967:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768023.731969:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfed80 x1862022294264832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768048 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768023.731977:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768023.731978:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768023.731979:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543dc00. 02000000:00000001:1.0:1775768023.731980:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768023.731981:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768023.731982:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:1.0:1775768023.731983:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768023.731984:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1f000. 02000000:00000001:1.0:1775768023.731985:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768023.731985:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768023.731986:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfed80. 00000100:00000001:1.0:1775768023.731987:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768023.731988:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768023.731989:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768023.731992:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768023.731992:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768023.731994:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768023.731995:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768023.793302:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768023.793310:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768023.793316:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768023.793318:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768023.793322:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768023.793323:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768023.793326:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768023.793327:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768023.793330:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768023.793331:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768023.793362:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768023.793365:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768023.793368:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768023.857270:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768023.857279:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768023.857282:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768023.857289:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768023.857294:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768023.857299:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768023.857302:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768023.857305:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768023.857309:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768024.753242:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768024.753245:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768024.753252:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768024.753252:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768024.753255:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768024.753256:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768024.753264:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775768024.753264:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775768024.753266:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768024.753273:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768024.753273:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768024.753273:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768024.753277:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768024.753279:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768024.753279:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768024.753282:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768024.753284:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768024.753286:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768024.753287:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768024.753289:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768024.753290:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768024.753292:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768024.753295:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768024.753298:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768024.753299:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768024.753303:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768024.753306:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768024.817294:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768024.817301:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768024.817306:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768024.817307:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768024.817311:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768024.817312:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768024.817315:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768024.817317:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768024.817320:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768024.817321:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768024.817342:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768024.817344:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768024.817346:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768024.881223:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768024.881231:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768024.881233:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768024.881239:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768024.881243:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768024.881247:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768024.881250:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768024.881252:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768024.881255:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768025.073244:0:14953:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4299949886, at 4299949888 00000100:00000040:1.0:1775768025.073244:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299949886, at 4299949888 00000100:00000010:0.0:1775768025.073261:0:14953:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cba9c0. 00000100:00000010:1.0:1775768025.073262:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cba120. 00000100:00000001:0.0:1775768025.777286:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768025.777286:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768025.777295:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768025.777295:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768025.777297:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768025.777298:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768025.777298:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768025.777303:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768025.777305:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768025.777306:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775768025.777306:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:1.0:1775768025.777315:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768025.777315:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768025.777316:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768025.777323:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768025.777323:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768025.777324:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768025.777330:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768025.777330:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768025.777340:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768025.777344:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768025.777345:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768025.777346:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768025.777349:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768025.777350:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768025.777353:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768025.777356:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768025.841238:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768025.841244:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768025.841251:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768025.841255:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768025.841258:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768025.841262:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768025.841265:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768025.841267:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768025.841269:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768025.841271:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768025.841274:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768025.841275:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768025.841276:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768025.841277:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768025.841278:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768025.841279:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768025.841280:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768025.841282:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768025.841284:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768025.841285:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768025.841286:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768025.841286:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768025.841287:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768025.841288:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768025.841289:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768025.841290:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768025.841292:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768025.841293:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768025.841294:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768025.841295:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768025.841296:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768025.841297:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768025.841298:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768025.841299:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768025.841314:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768025.841317:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768025.841319:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768025.905255:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768025.905263:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768025.905265:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768025.905271:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768025.905275:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768025.905279:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768025.905282:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768025.905285:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768025.905288:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768026.801217:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768026.801222:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768026.801223:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768026.801226:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768026.801226:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768026.801228:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768026.801229:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768026.801231:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:0.0:1775768026.801233:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:3.0:1775768026.801234:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775768026.801236:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768026.801237:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768026.801238:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:0.0:1775768026.801242:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768026.801242:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768026.801242:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768026.801247:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768026.801248:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768026.801250:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768026.801251:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768026.801254:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768026.801254:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768026.801257:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768026.801259:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768026.801260:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768026.801263:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768026.801265:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768026.865255:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768026.865261:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768026.865266:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768026.865267:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768026.865270:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768026.865271:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768026.865274:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768026.865275:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768026.865277:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768026.865278:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768026.865294:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768026.865296:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768026.865298:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768026.929267:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768026.929274:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768026.929276:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768026.929283:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768026.929287:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768026.929291:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768026.929294:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768026.929296:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768026.929310:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768027.825259:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768027.825259:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768027.825268:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768027.825268:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768027.825273:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768027.825273:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768027.825281:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:0.0:1775768027.825282:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768027.825289:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768027.825289:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768027.825295:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768027.825295:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768027.825300:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768027.825304:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768027.825307:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768027.825314:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768027.825317:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768027.825320:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768027.825404:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768027.825407:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768027.825408:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768027.825410:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768027.825412:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768027.825414:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768027.825416:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768027.825418:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768027.825420:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768027.889241:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768027.889247:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768027.889253:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768027.889254:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768027.889258:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768027.889260:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768027.889263:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768027.889264:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768027.889268:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768027.889269:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768027.889289:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768027.889292:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768027.889295:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768027.953236:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768027.953242:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768027.953245:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768027.953250:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768027.953254:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768027.953258:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768027.953261:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768027.953263:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768027.953266:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768028.849232:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768028.849239:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768028.849241:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768028.849248:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768028.849253:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768028.849257:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768028.849259:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768028.849262:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768028.849264:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768028.849292:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:2.0:1775768028.849292:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775768028.849299:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768028.849304:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768028.849305:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:2.0:1775768028.849306:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:0.0:1775768028.849308:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768028.849312:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768028.849313:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1775768028.849314:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768028.849314:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768028.849320:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00080000:2.0:1775768028.849320:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768028.849321:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768028.849328:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768028.849329:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768028.849335:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768028.849335:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775768028.849336:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768028.849344:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768028.849345:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768028.849348:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1775768028.849353:0:13471:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5281 00000100:00000001:1.0:1775768028.849354:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768028.849372:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768028.849380:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775768028.849384:0:13471:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775768028.849388:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768028.849392:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768028.849397:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768028.849400:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775768028.849405:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775768028.849408:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:2.0:1775768028.849410:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775768028.849414:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:2.0:1775768028.849419:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:2.0:1775768028.849422:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:2.0:1775768028.849427:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768028.849430:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768028.849435:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768028.849439:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000080:00000001:1.0:1775768028.849441:0:8:0:(rw.c:1481:ll_writepages()) Process entered 00000020:00000040:2.0:1775768028.849444:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000080:00000001:1.0:1775768028.849445:0:8:0:(file.c:5250:cl_sync_file_range()) Process entered 00000020:00000001:1.0:1775768028.849450:0:8:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00080000:2.0:1775768028.849451:0:13471:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000020:00000010:1.0:1775768028.849463:0:8:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994206aba240. 00000100:00000001:2.0:1775768028.849471:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1775768028.849472:0:8:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994201343000. 02000000:00000001:2.0:1775768028.849477:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000020:00000010:1.0:1775768028.849480:0:8:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994201343200. 02000000:00000001:2.0:1775768028.849481:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768028.849488:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1775768028.849489:0:8:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207cba120. 00000100:00000001:2.0:1775768028.849497:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00000010:1.0:1775768028.849500:0:8:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207e02ca0. 00000100:00000001:2.0:1775768028.849501:0:13471:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:1.0:1775768028.849509:0:8:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff9942031b3480. 00008000:00000010:1.0:1775768028.849515:0:8:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942054ab170. 00000100:00000010:2.0:1775768028.849515:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dff480. 00000020:00000040:2.0:1775768028.849523:0:13471:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 00000080:00000010:1.0:1775768028.849525:0:8:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99422030bf18. 02000000:00000001:2.0:1775768028.849529:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000080:00000010:1.0:1775768028.849530:0:8:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff994205751da8. 02000000:00000001:2.0:1775768028.849532:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1775768028.849538:0:8:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994201342e00. 00000100:00000001:2.0:1775768028.849538:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775768028.849545:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000008:00000010:1.0:1775768028.849546:0:8:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff994210709ce0. 00020000:00000010:1.0:1775768028.849553:0:8:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994205743180. 02000000:00000001:2.0:1775768028.849554:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:1.0:1775768028.849562:0:8:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994219fec110. 02000000:00000010:2.0:1775768028.849567:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1f400. 00000080:00000010:1.0:1775768028.849568:0:8:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942057e2960. 00000020:00000001:1.0:1775768028.849572:0:8:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107591643720 : -112966117907896 : ffff994206aba248) 00000020:00001000:1.0:1775768028.849574:0:8:0:(cl_object.c:831:cl_env_get()) 1@ffff994206aba240 00000020:00000001:1.0:1775768028.849583:0:8:0:(cl_object.c:151:cl_object_top()) ffff9942078a76c0 -> ffff9942078a76c0 00000100:00000001:2.0:1775768028.849585:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768028.849587:0:8:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:1.0:1775768028.849590:0:8:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:1.0:1775768028.849593:0:8:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:1.0:1775768028.849597:0:8:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:1.0:1775768028.849600:0:8:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00100000:2.0:1775768028.849601:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dff480 x1862022294264960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000080:00000001:1.0:1775768028.849602:0:8:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:1.0:1775768028.849613:0:8:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017a:0x0]io ffff994205751e08 type 6 ignore/verify layout 1/0 00020000:00000002:1.0:1775768028.849633:0:8:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207c1f9b0) owner 0000000000000000/ffff994201224f80 00020000:00000001:1.0:1775768028.849637:0:8:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:1.0:1775768028.849638:0:8:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:1.0:1775768028.849643:0:8:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00020000:00000001:1.0:1775768028.849647:0:8:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775768028.849651:0:8:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:1.0:1775768028.849655:0:8:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00000100:00000001:2.0:1775768028.849655:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00020000:00000001:1.0:1775768028.849659:0:8:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768028.849661:0:8:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:1.0:1775768028.849664:0:8:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00000001:2.0:1775768028.849665:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775768028.849669:0:8:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00020000:00000002:1.0:1775768028.849671:0:8:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207c1f9b0) owner 0000000000000000/ffff994201224f80 00000020:00000001:1.0:1775768028.849677:0:8:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768028.849677:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093a15639] to pc [ptlrpcd_rcv+-1] req@ffff994207dff480 x1862022294264960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1775768028.849683:0:8:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:1.0:1775768028.849687:0:8:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00020000:00000001:1.0:1775768028.849692:0:8:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00020000:00200000:1.0:1775768028.849698:0:8:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00000002:1.0:1775768028.849707:0:8:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:1.0:1775768028.849712:0:8:0:(lov_io.c:155:lov_sub_get()) Process entered 00020000:00000001:1.0:1775768028.849713:0:8:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000020:00000001:1.0:1775768028.849714:0:8:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:1.0:1775768028.849717:0:8:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994206ababd0. 00000020:00000010:1.0:1775768028.849720:0:8:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994208a8e000. 00000100:00000001:2.0:1775768028.849721:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1775768028.849723:0:8:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994208a8f600. 00000020:00000010:1.0:1775768028.849727:0:8:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207cba9c0. 00000100:00080000:2.0:1775768028.849728:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000008:00000010:1.0:1775768028.849736:0:8:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207e00000. 00000100:00080000:2.0:1775768028.849737:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00020000:00000010:1.0:1775768028.849742:0:8:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff9942031b30c0. 00000100:00000001:2.0:1775768028.849743:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1775768028.849748:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00008000:00000010:1.0:1775768028.849749:0:8:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942054a8a98. 00000100:00080000:2.0:1775768028.849749:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768028.849752:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768028.849760:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000080:00000010:1.0:1775768028.849761:0:8:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99422030b0a8. 00000100:00000001:2.0:1775768028.849763:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000080:00000010:1.0:1775768028.849767:0:8:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff994205753d98. 00000020:00000010:1.0:1775768028.849772:0:8:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994208a8ec00. 00000100:00000040:0.0:1775768028.849774:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dff480 x1862022294264960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768028.849775:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5281 00000008:00000010:1.0:1775768028.849781:0:8:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff994210709080. 00020000:00000010:1.0:1775768028.849788:0:8:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff9942057418c0. 00000400:00000200:2.0:1775768028.849795:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00008000:00000010:1.0:1775768028.849796:0:8:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994219fec368. 00000080:00000010:1.0:1775768028.849804:0:8:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942057e24b0. 00000100:00000001:0.0:1775768028.849807:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000001:1.0:1775768028.849808:0:8:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107591646168 : -112966117905448 : ffff994206ababd8) 00000100:00000001:0.0:1775768028.849810:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768028.849812:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000020:00001000:1.0:1775768028.849814:0:8:0:(cl_object.c:831:cl_env_get()) 1@ffff994206ababd0 02000000:00000001:0.0:1775768028.849818:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:1.0:1775768028.849821:0:8:0:(cl_object.c:151:cl_object_top()) ffff9942078a7788 -> ffff9942078a76c0 00000100:00080000:2.0:1775768028.849824:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 02000000:00000001:0.0:1775768028.849825:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1775768028.849825:0:8:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:1.0:1775768028.849832:0:8:0:(cl_io.c:898:cl_io_slice_add()) Process entered 02000000:00000001:0.0:1775768028.849833:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768028.849834:0:8:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:1.0:1775768028.849839:0:8:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768028.849839:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00100000:0.0:1775768028.849846:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294264960:192.168.202.156@tcp:250:kworker.0 00020000:00000001:1.0:1775768028.849846:0:8:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768028.849851:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00020000:00000001:1.0:1775768028.849853:0:8:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107571233256 : -112966138318360 : ffff9942057431e8) 00000100:00000001:0.0:1775768028.849860:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000001:1.0:1775768028.849862:0:8:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000100:00000040:2.0:1775768028.849864:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000001:1.0:1775768028.849866:0:8:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768028.849868:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00020000:00000001:1.0:1775768028.849874:0:8:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768028.849875:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768028.849875:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775768028.849884:0:8:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768028.849885:0:13471:0:(genops.c:811:class_conn2export()) Process entered 02000000:00000001:0.0:1775768028.849886:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:1.0:1775768028.849887:0:8:0:(cl_io.c:314:cl_io_lock()) Process entered 00020000:00000001:1.0:1775768028.849895:0:8:0:(lov_io.c:1108:lov_io_lock()) Process entered 00000020:00000040:2.0:1775768028.849895:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 02000000:00000010:0.0:1775768028.849899:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220028000. 00020000:00000001:1.0:1775768028.849901:0:8:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775768028.849901:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000001:0.0:1775768028.849905:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768028.849905:0:8:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000040:2.0:1775768028.849907:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:1.0:1775768028.849908:0:8:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:1.0:1775768028.849912:0:8:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768028.849914:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:1.0:1775768028.849915:0:8:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768028.849922:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:1.0:1775768028.849924:0:8:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768028.849928:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00020000:00000001:1.0:1775768028.849929:0:8:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775768028.849930:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000020:00000001:1.0:1775768028.849938:0:8:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:1.0:1775768028.849941:0:8:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768028.849942:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000010:0.0:1775768028.849944:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000020:00000001:1.0:1775768028.849946:0:8:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768028.849953:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:1.0:1775768028.849954:0:8:0:(cl_io.c:528:cl_io_start()) Process entered 00000100:00000200:0.0:1775768028.849959:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294264960, portal 25 00020000:00000001:1.0:1775768028.849960:0:8:0:(lov_io.c:1114:lov_io_start()) Process entered 00020000:00000001:1.0:1775768028.849965:0:8:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000040:2.0:1775768028.849967:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000001:1.0:1775768028.849968:0:8:0:(cl_io.c:528:cl_io_start()) Process entered 00000100:00000001:0.0:1775768028.849969:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768028.849975:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611849856 : -112966097701760 : ffff994207dff480) 00000008:00000001:1.0:1775768028.849975:0:8:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000100:00000001:2.0:1775768028.849976:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000008:00000001:1.0:1775768028.849979:0:8:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000100:00000040:0.0:1775768028.849986:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dff480 x1862022294264960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768053 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1775768028.849987:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000008:00000001:1.0:1775768028.849992:0:8:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:1.0:1775768028.849998:0:8:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000100:00080000:2.0:1775768028.850003:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000008:00000001:1.0:1775768028.850004:0:8:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:1.0:1775768028.850013:0:8:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000100:00000001:0.0:1775768028.850014:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00000001:1.0:1775768028.850017:0:8:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000100:00000001:2.0:1775768028.850019:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1775768028.850020:0:8:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107884691392 : -112965824860224 : ffff994218232fc0) 00000100:00000040:0.0:1775768028.850027:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 02000000:00000001:2.0:1775768028.850030:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:0.0:1775768028.850031:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000008:00000020:1.0:1775768028.850031:0:8:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994218232fc0 ready 0|-|- wr 1|-|+ rd 0|- 10 in flight 00000008:00000001:1.0:1775768028.850039:0:8:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000100:00000200:0.0:1775768028.850040:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294264960, offset 0 00000008:00000020:1.0:1775768028.850041:0:8:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 02000000:00000001:2.0:1775768028.850043:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00000001:1.0:1775768028.850045:0:8:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:1.0:1775768028.850052:0:8:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000400:00000200:0.0:1775768028.850054:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 02000000:00000001:2.0:1775768028.850054:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000008:00000020:1.0:1775768028.850056:0:8:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994210675880 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000100:00000001:2.0:1775768028.850065:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775768028.850072:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000400:00000200:0.0:1775768028.850075:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00000001:1.0:1775768028.850077:0:8:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1775768028.850085:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1775768028.850087:0:8:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768028.850092:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00000001:2.0:1775768028.850093:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768028.850094:0:8:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775768028.850104:0:8:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775768028.850106:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfce00. 00000400:00000200:0.0:1775768028.850107:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00020000:00000001:1.0:1775768028.850108:0:8:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768028.850116:0:8:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775768028.850117:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000400:00000200:0.0:1775768028.850121:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000020:00000001:1.0:1775768028.850123:0:8:0:(cl_io.c:555:cl_io_end()) Process entered 00020000:00000001:1.0:1775768028.850128:0:8:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 02000000:00000001:2.0:1775768028.850129:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000200:0.0:1775768028.850130:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00020000:00000001:1.0:1775768028.850131:0:8:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 02000000:00000001:2.0:1775768028.850133:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768028.850135:0:8:0:(cl_io.c:555:cl_io_end()) Process entered 00000400:00000200:0.0:1775768028.850138:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000020:00000001:1.0:1775768028.850139:0:8:0:(cl_io.c:563:cl_io_end()) Process leaving 00000100:00000001:2.0:1775768028.850140:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00020000:00000001:1.0:1775768028.850142:0:8:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768028.850144:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00020000:00000001:1.0:1775768028.850146:0:8:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000400:00000200:0.0:1775768028.850149:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225243:225243:256:4294967295] 192.168.202.156@tcp LPNI seq info [225239:225239:8:4294967295] 00000020:00000001:1.0:1775768028.850149:0:8:0:(cl_io.c:563:cl_io_end()) Process leaving 02000000:00000001:2.0:1775768028.850150:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768028.850153:0:8:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000400:00000200:0.0:1775768028.850180:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000010:2.0:1775768028.850191:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1d000. 00000400:00000200:0.0:1775768028.850198:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00020000:00000001:1.0:1775768028.850204:0:8:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00000100:00000001:2.0:1775768028.850205:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775768028.850206:0:8:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:1.0:1775768028.850211:0:8:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000800:00000200:0.0:1775768028.850216:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000020:00000001:1.0:1775768028.850216:0:8:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:1.0:1775768028.850220:0:8:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768028.850223:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfce00 x1862022294265088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000010:0.0:1775768028.850224:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00020000:00000001:1.0:1775768028.850224:0:8:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775768028.850235:0:8:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000020:00000001:1.0:1775768028.850240:0:8:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000800:00000200:0.0:1775768028.850245:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000001:1.0:1775768028.850246:0:8:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000020:00000001:1.0:1775768028.850251:0:8:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000800:00000200:0.0:1775768028.850256:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000020:00000001:1.0:1775768028.850257:0:8:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:1.0:1775768028.850263:0:8:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00020000:00000001:1.0:1775768028.850269:0:8:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:1.0:1775768028.850272:0:8:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000100:00000001:2.0:1775768028.850272:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000020:00000001:1.0:1775768028.850276:0:8:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000800:00000200:0.1:1775768028.850278:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00020000:00000001:1.0:1775768028.850281:0:8:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768028.850283:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775768028.850285:0:8:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775768028.850286:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00020000:00000001:1.0:1775768028.850290:0:8:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000020:00000001:1.0:1775768028.850293:0:8:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000100:00000040:2.0:1775768028.850294:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e5a7debe] to pc [ptlrpcd_rcv+-1] req@ffff994207dfce00 x1862022294265088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:0.0:1775768028.850295:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000020:00000001:1.0:1775768028.850295:0:8:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768028.850303:0:8:0:(cl_io.c:71:cl_io_fini()) Process entered 00000800:00000010:0.0:1775768028.850304:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00020000:00000001:1.0:1775768028.850306:0:8:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:1.0:1775768028.850311:0:8:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000400:00000200:0.0:1775768028.850316:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000001:1.0:1775768028.850316:0:8:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:1.0:1775768028.850319:0:8:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:1.0:1775768028.850325:0:8:0:(cl_object.c:925:cl_env_put()) 1@ffff994206ababd0 00000020:00000010:1.0:1775768028.850333:0:8:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994208a8f600. 00000400:00000200:0.0:1775768028.850334:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000020:00000010:1.0:1775768028.850347:0:8:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207cba9c0. 00000400:00000200:0.0:1775768028.850348:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000100:00000001:2.0:1775768028.850348:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:1.0:1775768028.850353:0:8:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207e00000. 00000400:00000010:0.0:1775768028.850354:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00080000:2.0:1775768028.850358:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00020000:00000010:1.0:1775768028.850361:0:8:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff9942031b30c0. 00000100:00000001:0.0:1775768028.850362:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:2.0:1775768028.850367:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00008000:00000010:1.0:1775768028.850369:0:8:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942054a8a98. 00000100:00000001:2.0:1775768028.850370:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000200:0.0:1775768028.850372:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dff480 x1862022294264960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768053 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000010:1.0:1775768028.850376:0:8:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99422030b0a8. 00000100:00080000:2.0:1775768028.850376:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000080:00000010:1.0:1775768028.850386:0:8:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff994205753d98. 00000100:00000001:2.0:1775768028.850390:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000020:00000010:1.0:1775768028.850391:0:8:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994208a8e000. 00000100:00000001:0.0:1775768028.850393:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000008:00000010:1.0:1775768028.850395:0:8:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff994210709080. 00000100:00080000:2.0:1775768028.850399:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5281 00000100:00000040:0.0:1775768028.850401:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dff480 x1862022294264960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768053 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000010:1.0:1775768028.850403:0:8:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff9942057418c0. 00008000:00000010:1.0:1775768028.850416:0:8:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994219fec368. 00000100:00000001:0.0:1775768028.850422:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768028.850424:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000080:00000010:1.0:1775768028.850426:0:8:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942057e24b0. 00000100:00000001:0.0:1775768028.850429:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000010:1.0:1775768028.850435:0:8:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994208a8ec00. 00000100:00000001:0.0:1775768028.850438:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768028.850439:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000020:00000010:1.0:1775768028.850441:0:8:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994206ababd0. 00000100:00080000:2.0:1775768028.850450:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000400:00000200:0.0:1775768028.850451:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:1.0:1775768028.850451:0:8:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:1.0:1775768028.850456:0:8:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:1.0:1775768028.850462:0:8:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000100:00000001:0.0:1775768028.850464:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299955006 00000080:00200000:1.0:1775768028.850465:0:8:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00000040:2.0:1775768028.850470:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000080:00000001:1.0:1775768028.850472:0:8:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000100:00000010:0.0:1775768028.850473:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942078a3d80. 00000020:00000001:1.0:1775768028.850477:0:8:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000100:00000001:2.0:1775768028.850478:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00001000:1.0:1775768028.850482:0:8:0:(cl_object.c:925:cl_env_put()) 1@ffff994206aba240 00000100:00000040:2.0:1775768028.850485:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000010:1.0:1775768028.850486:0:8:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994201343200. 00000100:00000001:2.0:1775768028.850492:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:0.0:1775768028.850494:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000020:00000010:1.0:1775768028.850495:0:8:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207cba120. 00000020:00000001:2.0:1775768028.850497:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775768028.850502:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000008:00000010:1.0:1775768028.850503:0:8:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207e02ca0. 00000100:00000001:0.0:1775768028.850504:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299955006 00000020:00000001:2.0:1775768028.850512:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000010:0.0:1775768028.850513:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942078a3c00. 00020000:00000010:1.0:1775768028.850513:0:8:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff9942031b3480. 00000020:00000040:2.0:1775768028.850515:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000100:00000001:0.0:1775768028.850520:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00008000:00000010:1.0:1775768028.850521:0:8:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942054ab170. 00000020:00000001:2.0:1775768028.850521:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000080:00000010:1.0:1775768028.850530:0:8:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99422030bf18. 00000020:00000001:2.0:1775768028.850532:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000001:0.0:1775768028.850533:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000010:1.0:1775768028.850536:0:8:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff994205751da8. 00000100:00000001:0.0:1775768028.850543:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768028.850547:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000020:00000010:1.0:1775768028.850548:0:8:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994201343000. 00000100:00000040:2.0:1775768028.850549:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000008:00000010:1.0:1775768028.850560:0:8:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff994210709ce0. 00000100:00100000:0.0:1775768028.850561:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768028/real 1775768028] req@ffff994207dff480 x1862022294264960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768028.850561:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00020000:00000010:1.0:1775768028.850570:0:8:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994205743180. 00008000:00000010:1.0:1775768028.850578:0:8:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994219fec110. 00000100:00000040:2.0:1775768028.850578:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000080:00000010:1.0:1775768028.850586:0:8:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942057e2960. 00000020:00000010:1.0:1775768028.850591:0:8:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994201342e00. 00000100:00000001:2.0:1775768028.850591:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000001:0.0:1775768028.850592:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768028.850596:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000010:1.0:1775768028.850597:0:8:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994206aba240. 00000080:00000001:1.0:1775768028.850604:0:8:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:2.0:1775768028.850604:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000400:00000010:0.0:1775768028.850605:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775768028.850628:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000100:00080000:2.0:1775768028.850631:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000400:00000010:0.0:1775768028.850632:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000080:00000001:1.0:1775768028.850634:0:8:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768028.850642:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1775768028.850643:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1775768028.850647:0:8:0:(rw.c:1481:ll_writepages()) Process entered 02000000:00000001:2.0:1775768028.850648:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000200:0.0:1775768028.850649:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dff480 x1862022294264960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000001:1.0:1775768028.850653:0:8:0:(file.c:5250:cl_sync_file_range()) Process entered 00000020:00000001:1.0:1775768028.850656:0:8:0:(cl_object.c:762:cl_env_obtain()) Process entered 02000000:00000001:2.0:1775768028.850657:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1775768028.850662:0:8:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994206aba240. 02000000:00000001:2.0:1775768028.850668:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1775768028.850670:0:8:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994208a8ec00. 00000020:00000010:1.0:1775768028.850680:0:8:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994208a8e000. 00000100:00000001:2.0:1775768028.850681:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000200:0.0:1775768028.850687:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dff480 x1862022294264960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1775768028.850687:0:8:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207cba120. 00000008:00100000:2.0:1775768028.850688:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000010:1.0:1775768028.850692:0:8:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207e02ca0. 00020000:00000010:1.0:1775768028.850699:0:8:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff9942031b3480. 00000008:00000001:2.0:1775768028.850703:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:1.0:1775768028.850705:0:8:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942054ab170. 00000080:00000010:1.0:1775768028.850716:0:8:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99422030bf18. 00000100:00000001:2.0:1775768028.850717:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768028.850718:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768028.850722:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000080:00000010:1.0:1775768028.850722:0:8:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff994205751da8. 00000100:00000001:0.0:1775768028.850727:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000010:1.0:1775768028.850729:0:8:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994208a8f600. 00000100:00000010:2.0:1775768028.850729:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfed80. 00000100:00000001:0.0:1775768028.850730:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00000010:1.0:1775768028.850739:0:8:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff994210709ce0. 00000020:00000040:2.0:1775768028.850740:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000100:00100000:0.0:1775768028.850744:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dff480 x1862022294264960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000010:1.0:1775768028.850744:0:8:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994205743180. 02000000:00000001:2.0:1775768028.850752:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00008000:00000010:1.0:1775768028.850753:0:8:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994219fec110. 02000000:00000001:2.0:1775768028.850759:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:1.0:1775768028.850760:0:8:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942057e2960. 00000020:00000001:1.0:1775768028.850770:0:8:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107591643720 : -112966117907896 : ffff994206aba248) 00000100:00000001:2.0:1775768028.850771:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1775768028.850772:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1775768028.850778:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000020:00001000:1.0:1775768028.850779:0:8:0:(cl_object.c:831:cl_env_get()) 1@ffff994206aba240 00000020:00000001:1.0:1775768028.850783:0:8:0:(cl_object.c:151:cl_object_top()) ffff9942078a7f58 -> ffff9942078a7f58 00000100:00000040:0.0:1775768028.850784:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dff480 x1862022294264960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768053 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1775768028.850784:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768028.850790:0:8:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:1.0:1775768028.850794:0:8:0:(vvp_io.c:1812:vvp_io_init()) Process entered 02000000:00000010:2.0:1775768028.850796:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1f800. 00000080:00200000:1.0:1775768028.850798:0:8:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017b:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000100:00000001:2.0:1775768028.850806:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768028.850807:0:8:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000100:00000001:0.0:1775768028.850812:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:1.0:1775768028.850812:0:8:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:1.0:1775768028.850814:0:8:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768028.850819:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00020000:00000002:1.0:1775768028.850821:0:8:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017b:0x0]io ffff994205751e08 type 6 ignore/verify layout 1/0 00000100:00000001:0.0:1775768028.850822:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00100000:2.0:1775768028.850822:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfed80 x1862022294265216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768028.850826:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00020000:00000002:1.0:1775768028.850833:0:8:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207c1f000) owner 0000000000000000/ffff994201224f80 00000100:00000001:0.0:1775768028.850836:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00020000:00000001:1.0:1775768028.850840:0:8:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00000100:00000001:0.0:1775768028.850842:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00020000:00000001:1.0:1775768028.850845:0:8:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:1.0:1775768028.850848:0:8:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00000100:00080000:0.0:1775768028.850854:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00020000:00000001:1.0:1775768028.850854:0:8:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775768028.850864:0:8:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768028.850869:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00020000:00000001:1.0:1775768028.850870:0:8:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00000100:00080000:0.0:1775768028.850874:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00020000:00000001:1.0:1775768028.850874:0:8:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768028.850878:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768028.850881:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000001:1.0:1775768028.850881:0:8:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:1.0:1775768028.850885:0:8:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:1.0:1775768028.850887:0:8:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00000100:00000040:0.0:1775768028.850891:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dff480 x1862022294264960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768053 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000002:1.0:1775768028.850892:0:8:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207c1f000) owner 0000000000000000/ffff994201224f80 00000100:00000040:2.0:1775768028.850892:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb9b9eae] to pc [ptlrpcd_rcv+-1] req@ffff994207dfed80 x1862022294265216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1775768028.850903:0:8:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768028.850912:0:8:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:1.0:1775768028.850917:0:8:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00020000:00000001:1.0:1775768028.850920:0:8:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00020000:00200000:1.0:1775768028.850925:0:8:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00000002:1.0:1775768028.850932:0:8:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00000100:00100000:0.0:1775768028.850942:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294264960:192.168.202.156@tcp:250:kworker.0 00020000:00000001:1.0:1775768028.850942:0:8:0:(lov_io.c:155:lov_sub_get()) Process entered 00020000:00000001:1.0:1775768028.850945:0:8:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000020:00000001:1.0:1775768028.850950:0:8:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:1.0:1775768028.850957:0:8:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994206ababd0. 00000100:00000001:2.0:1775768028.850960:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768028.850961:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:1.0:1775768028.850962:0:8:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994201342e00. 00000020:00000010:1.0:1775768028.850970:0:8:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994201343000. 00000100:00000001:0.0:1775768028.850971:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00080000:2.0:1775768028.850972:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000010:1.0:1775768028.850974:0:8:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207cba9c0. 00000100:00000040:0.0:1775768028.850978:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dff480 x1862022294264960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768053 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000010:1.0:1775768028.850983:0:8:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207e00000. 00000100:00080000:2.0:1775768028.850989:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00020000:00000010:1.0:1775768028.850990:0:8:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff9942031b30c0. 00008000:00000010:1.0:1775768028.850997:0:8:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942054a8a98. 00000100:00000001:2.0:1775768028.850997:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1775768028.850999:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 00000100:00080000:2.0:1775768028.851004:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:0.0:1775768028.851005:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000080:00000010:1.0:1775768028.851005:0:8:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99422030b0a8. 02000000:00000010:0.0:1775768028.851013:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220028000. 00000080:00000010:1.0:1775768028.851013:0:8:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff994205753d98. 00000100:00000001:2.0:1775768028.851015:0:13471:0:(import.c:484:import_select_connection()) Process entered 02000000:00000001:0.0:1775768028.851024:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000010:1.0:1775768028.851024:0:8:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994201343200. 00000020:00000001:0.0:1775768028.851030:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000008:00000010:1.0:1775768028.851031:0:8:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff994210709080. 00000100:00080000:2.0:1775768028.851031:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5281 00000020:00000040:0.0:1775768028.851034:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00020000:00000010:1.0:1775768028.851036:0:8:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff9942057418c0. 00000020:00000001:0.0:1775768028.851044:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 00008000:00000010:1.0:1775768028.851044:0:8:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994219fec368. 02000000:00000010:0.0:1775768028.851050:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1f400. 00000400:00000200:2.0:1775768028.851052:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000080:00000010:1.0:1775768028.851055:0:8:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942057e24b0. 02000000:00000001:0.0:1775768028.851057:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768028.851061:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768028.851064:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dff480. 00000020:00000001:1.0:1775768028.851064:0:8:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107591646168 : -112966117905448 : ffff994206ababd8) 00000400:00000200:2.0:1775768028.851065:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:0.0:1775768028.851072:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000020:00001000:1.0:1775768028.851075:0:8:0:(cl_object.c:831:cl_env_get()) 1@ffff994206ababd0 00000100:00000001:0.0:1775768028.851077:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1775768028.851078:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000020:00000001:1.0:1775768028.851080:0:8:0:(cl_object.c:151:cl_object_top()) ffff9942078a75f8 -> ffff9942078a7f58 00000100:00000001:0.0:1775768028.851081:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1775768028.851088:0:8:0:(cl_io.c:125:__cl_io_init()) Process entered 00000100:00000040:2.0:1775768028.851090:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768028.851091:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:1.0:1775768028.851092:0:8:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000100:00000001:0.0:1775768028.851095:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768028.851099:0:8:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00000001:2.0:1775768028.851101:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1775768028.851102:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1775768028.851103:0:8:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768028.851108:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00020000:00000001:1.0:1775768028.851108:0:8:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768028.851110:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768028.851112:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00020000:00000001:1.0:1775768028.851117:0:8:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107571233256 : -112966138318360 : ffff9942057431e8) 00000100:00000001:2.0:1775768028.851120:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000040:0.0:1775768028.851122:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfce00 x1862022294265088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1775768028.851125:0:8:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000020:00000001:2.0:1775768028.851128:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000001:1.0:1775768028.851129:0:8:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775768028.851136:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00020000:00000001:1.0:1775768028.851138:0:8:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768028.851143:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000001:1.0:1775768028.851145:0:8:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775768028.851149:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:1.0:1775768028.851203:0:8:0:(cl_io.c:314:cl_io_lock()) Process entered 00000100:00000001:0.0:1775768028.851205:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000001:2.0:1775768028.851206:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00020000:00000001:1.0:1775768028.851207:0:8:0:(lov_io.c:1108:lov_io_lock()) Process entered 00000100:00000001:0.0:1775768028.851208:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775768028.851212:0:8:0:(lov_io.c:1086:lov_io_call()) Process entered 02000000:00000001:0.0:1775768028.851215:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:2.0:1775768028.851217:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:1.0:1775768028.851219:0:8:0:(cl_io.c:314:cl_io_lock()) Process entered 02000000:00000001:0.0:1775768028.851220:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1775768028.851223:0:8:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:1.0:1775768028.851229:0:8:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768028.851230:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768028.851236:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000001:1.0:1775768028.851237:0:8:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768028.851241:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294265088:192.168.202.156@tcp:38:kworker.0 00020000:00000001:1.0:1775768028.851250:0:8:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768028.851250:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1775768028.851252:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768028.851257:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00020000:00000001:1.0:1775768028.851257:0:8:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768028.851261:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000001:0.0:1775768028.851262:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768028.851266:0:8:0:(cl_io.c:286:cl_lockset_lock()) Process entered 02000000:00000001:0.0:1775768028.851271:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:1.0:1775768028.851273:0:8:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768028.851274:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 02000000:00000010:0.0:1775768028.851275:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220028000. 02000000:00000001:0.0:1775768028.851284:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768028.851285:0:8:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775768028.851286:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000020:00000001:1.0:1775768028.851295:0:8:0:(cl_io.c:528:cl_io_start()) Process entered 00000400:00000200:0.0:1775768028.851296:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:1.0:1775768028.851300:0:8:0:(lov_io.c:1114:lov_io_start()) Process entered 00000100:00080000:2.0:1775768028.851301:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000400:00000010:0.0:1775768028.851304:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00020000:00000001:1.0:1775768028.851304:0:8:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:1.0:1775768028.851310:0:8:0:(cl_io.c:528:cl_io_start()) Process entered 00000100:00000001:2.0:1775768028.851310:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775768028.851312:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000008:00000001:1.0:1775768028.851316:0:8:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000008:00000001:1.0:1775768028.851321:0:8:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000100:00000200:0.0:1775768028.851323:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294265088, portal 10 02000000:00000001:2.0:1775768028.851325:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000008:00000001:1.0:1775768028.851327:0:8:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:1.0:1775768028.851333:0:8:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000100:00000001:0.0:1775768028.851334:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1775768028.851336:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00000001:1.0:1775768028.851337:0:8:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768028.851338:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611840000 : -112966097711616 : ffff994207dfce00) 00000008:00000001:1.0:1775768028.851348:0:8:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 02000000:00000001:2.0:1775768028.851348:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768028.851352:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfce00 x1862022294265088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768053 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:1.0:1775768028.851354:0:8:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000008:00000001:1.0:1775768028.851358:0:8:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107884688256 : -112965824863360 : ffff994218232380) 00000100:00000001:2.0:1775768028.851358:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00000020:1.0:1775768028.851365:0:8:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994218232380 ready 0|-|- wr 1|-|+ rd 0|- 9 in flight 00000100:00000001:0.0:1775768028.851368:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00000001:1.0:1775768028.851369:0:8:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:1.0:1775768028.851371:0:8:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000100:00000040:0.0:1775768028.851373:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000008:00000001:1.0:1775768028.851374:0:8:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1775768028.851377:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000008:00000001:1.0:1775768028.851377:0:8:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000008:00000020:1.0:1775768028.851380:0:8:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994210674fc0 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000100:00000200:0.0:1775768028.851381:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294265088, offset 0 00000008:00100000:2.0:1775768028.851382:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:1.0:1775768028.851394:0:8:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1775768028.851394:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768028.851396:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000008:00000001:1.0:1775768028.851404:0:8:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768028.851405:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768028.851412:0:8:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768028.851413:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:1.0:1775768028.851416:0:8:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775768028.851417:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfe300. 00020000:00000001:1.0:1775768028.851418:0:8:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768028.851420:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000020:00000001:1.0:1775768028.851427:0:8:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775768028.851428:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00000400:00000200:0.0:1775768028.851429:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:1.0:1775768028.851432:0:8:0:(cl_io.c:555:cl_io_end()) Process entered 00020000:00000001:1.0:1775768028.851437:0:8:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00000400:00000200:0.0:1775768028.851438:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00020000:00000001:1.0:1775768028.851443:0:8:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 02000000:00000001:2.0:1775768028.851444:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000200:0.0:1775768028.851446:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000020:00000001:1.0:1775768028.851446:0:8:0:(cl_io.c:555:cl_io_end()) Process entered 02000000:00000001:2.0:1775768028.851448:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768028.851449:0:8:0:(cl_io.c:563:cl_io_end()) Process leaving 00000400:00000200:0.0:1775768028.851453:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00020000:00000001:1.0:1775768028.851453:0:8:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768028.851457:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00020000:00000001:1.0:1775768028.851458:0:8:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000020:00000001:1.0:1775768028.851463:0:8:0:(cl_io.c:563:cl_io_end()) Process leaving 00000400:00000200:0.0:1775768028.851464:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225244:225244:256:4294967295] 192.168.202.156@tcp LPNI seq info [225240:225240:8:4294967295] 02000000:00000001:2.0:1775768028.851465:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000020:00000001:1.0:1775768028.851466:0:8:0:(cl_io.c:352:cl_io_unlock()) Process entered 00020000:00000001:1.0:1775768028.851469:0:8:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00020000:00000001:1.0:1775768028.851471:0:8:0:(lov_io.c:1086:lov_io_call()) Process entered 02000000:00000001:2.0:1775768028.851473:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768028.851474:0:8:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000400:00000200:0.0:1775768028.851475:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:1.0:1775768028.851477:0:8:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:1.0:1775768028.851479:0:8:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775768028.851485:0:8:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775768028.851488:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1e400. 00020000:00000001:1.0:1775768028.851489:0:8:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000020:00000001:1.0:1775768028.851491:0:8:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000400:00000200:0.0:1775768028.851495:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000020:00000001:1.0:1775768028.851496:0:8:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000100:00000001:2.0:1775768028.851498:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768028.851499:0:8:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000001:1.0:1775768028.851502:0:8:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:1.0:1775768028.851505:0:8:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00020000:00000001:1.0:1775768028.851511:0:8:0:(lov_io.c:1086:lov_io_call()) Process entered 00000800:00000200:0.0:1775768028.851512:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000020:00000001:1.0:1775768028.851513:0:8:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000800:00000010:0.0:1775768028.851518:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000020:00000001:1.0:1775768028.851518:0:8:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000100:00100000:2.0:1775768028.851519:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfe300 x1862022294265344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00020000:00000001:1.0:1775768028.851521:0:8:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775768028.851530:0:8:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1775768028.851532:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00020000:00000001:1.0:1775768028.851537:0:8:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000800:00000200:0.0:1775768028.851543:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000020:00000001:1.0:1775768028.851543:0:8:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000020:00000001:1.0:1775768028.851549:0:8:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768028.851558:0:8:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:1.0:1775768028.851563:0:8:0:(lov_io.c:675:lov_io_fini()) Process entered 00000800:00000200:0.1:1775768028.851566:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00020000:00000001:1.0:1775768028.851568:0:8:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:1.0:1775768028.851572:0:8:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:1.0:1775768028.851577:0:8:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000800:00000200:0.1:1775768028.851578:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000020:00001000:1.0:1775768028.851582:0:8:0:(cl_object.c:925:cl_env_put()) 1@ffff994206ababd0 00000100:00000001:2.0:1775768028.851584:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000020:00000010:1.0:1775768028.851588:0:8:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994201343000. 00000800:00000100:0.0:1775768028.851590:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000100:00000001:2.0:1775768028.851591:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1775768028.851596:0:8:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207cba9c0. 00000800:00000010:0.0:1775768028.851597:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000008:00000010:1.0:1775768028.851601:0:8:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207e00000. 00000400:00000200:0.0:1775768028.851603:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:2.0:1775768028.851605:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c661fef] to pc [ptlrpcd_rcv+-1] req@ffff994207dfe300 x1862022294265344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000010:1.0:1775768028.851607:0:8:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff9942031b30c0. 00008000:00000010:1.0:1775768028.851629:0:8:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942054a8a98. 00000400:00000200:0.0:1775768028.851631:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000080:00000010:1.0:1775768028.851638:0:8:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99422030b0a8. 00000080:00000010:1.0:1775768028.851644:0:8:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff994205753d98. 00000400:00000200:0.0:1775768028.851650:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000020:00000010:1.0:1775768028.851653:0:8:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994201342e00. 00000400:00000010:0.0:1775768028.851657:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000008:00000010:1.0:1775768028.851661:0:8:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff994210709080. 00000100:00000001:0.0:1775768028.851665:0:2399:0:(events.c:34:request_out_callback()) Process entered 00020000:00000010:1.0:1775768028.851670:0:8:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff9942057418c0. 00000100:00000200:0.0:1775768028.851676:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfce00 x1862022294265088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768053 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00008000:00000010:1.0:1775768028.851680:0:8:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994219fec368. 00000100:00000001:2.0:1775768028.851683:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:1.0:1775768028.851685:0:8:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942057e24b0. 00000100:00000001:0.0:1775768028.851686:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000020:00000010:1.0:1775768028.851693:0:8:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994201343200. 00000100:00000040:0.0:1775768028.851694:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfce00 x1862022294265088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768053 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768028.851695:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000010:1.0:1775768028.851698:0:8:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994206ababd0. 00020000:00000001:1.0:1775768028.851706:0:8:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:1.0:1775768028.851711:0:8:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000100:00080000:2.0:1775768028.851714:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000080:00000001:1.0:1775768028.851715:0:8:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000100:00000001:2.0:1775768028.851721:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1775768028.851722:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000080:00200000:1.0:1775768028.851722:0:8:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00000001:0.0:1775768028.851730:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000080:00000001:1.0:1775768028.851731:0:8:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000100:00080000:2.0:1775768028.851731:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000020:00000001:1.0:1775768028.851737:0:8:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000100:00000001:0.0:1775768028.851739:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768028.851743:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000040:0.0:1775768028.851744:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000020:00001000:1.0:1775768028.851744:0:8:0:(cl_object.c:925:cl_env_put()) 1@ffff994206aba240 00000020:00000010:1.0:1775768028.851750:0:8:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994208a8e000. 00000100:00080000:2.0:1775768028.851755:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5281 00000400:00000200:0.0:1775768028.851756:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000010:1.0:1775768028.851758:0:8:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207cba120. 00000008:00000010:1.0:1775768028.851764:0:8:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207e02ca0. 00000100:00000001:0.0:1775768028.851765:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299953684 00020000:00000010:1.0:1775768028.851769:0:8:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff9942031b3480. 00000100:00000001:0.0:1775768028.851771:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299955006 00000400:00000200:2.0:1775768028.851772:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00008000:00000010:1.0:1775768028.851775:0:8:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942054ab170. 00000100:00000001:0.0:1775768028.851782:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299953684 00000100:00000001:0.0:1775768028.851786:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299955006 00000080:00000010:1.0:1775768028.851786:0:8:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99422030bf18. 00000400:00000200:2.0:1775768028.851792:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:0.0:1775768028.851793:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000010:1.0:1775768028.851793:0:8:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff994205751da8. 00000100:00000001:0.0:1775768028.851805:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1775768028.851806:0:8:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994208a8ec00. 00000100:00080000:2.0:1775768028.851806:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1775768028.851815:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000008:00000010:1.0:1775768028.851816:0:8:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff994210709ce0. 00000100:00000040:2.0:1775768028.851823:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00100000:0.0:1775768028.851824:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768028/real 1775768028] req@ffff994207dfce00 x1862022294265088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000010:1.0:1775768028.851825:0:8:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994205743180. 00008000:00000010:1.0:1775768028.851834:0:8:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994219fec110. 00000100:00000001:2.0:1775768028.851834:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000080:00000010:1.0:1775768028.851843:0:8:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942057e2960. 00000100:00000040:2.0:1775768028.851848:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000010:1.0:1775768028.851850:0:8:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994208a8f600. 00000100:00000001:2.0:1775768028.851859:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000010:1.0:1775768028.851860:0:8:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994206aba240. 00000400:00000001:0.0:1775768028.851864:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000080:00000001:1.0:1775768028.851867:0:8:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:0.0:1775768028.851869:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:2.0:1775768028.851869:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000080:00000001:1.0:1775768028.851870:0:8:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775768028.851876:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000020:00000040:2.0:1775768028.851877:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000080:00000001:1.0:1775768028.851878:0:8:0:(rw.c:1481:ll_writepages()) Process entered 00000080:00000001:1.0:1775768028.851881:0:8:0:(file.c:5250:cl_sync_file_range()) Process entered 00000020:00000001:2.0:1775768028.851887:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:0.0:1775768028.851888:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000020:00000001:1.0:1775768028.851888:0:8:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000400:00000010:0.0:1775768028.851895:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000020:00000040:2.0:1775768028.851896:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000010:1.0:1775768028.851897:0:8:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994206aba240. 00000100:00000001:0.0:1775768028.851901:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000010:1.0:1775768028.851907:0:8:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994201343200. 00000020:00000001:2.0:1775768028.851908:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000200:0.0:1775768028.851909:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfce00 x1862022294265088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1775768028.851914:0:8:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994201342e00. 00000020:00000001:2.0:1775768028.851918:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000010:1.0:1775768028.851920:0:8:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207cba120. 00000008:00000010:1.0:1775768028.851928:0:8:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207e02ca0. 00020000:00000010:1.0:1775768028.851934:0:8:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff9942031b3480. 00000100:00000040:2.0:1775768028.851936:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00008000:00000010:1.0:1775768028.851940:0:8:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942054ab170. 00000080:00000010:1.0:1775768028.851948:0:8:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99422030bf18. 00000100:00000001:2.0:1775768028.851948:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:0.0:1775768028.851954:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfce00 x1862022294265088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000010:1.0:1775768028.851956:0:8:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff994205751da8. 00000100:00000040:2.0:1775768028.851962:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000010:1.0:1775768028.851964:0:8:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994201343000. 00000008:00000010:1.0:1775768028.851976:0:8:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff994210709ce0. 00000100:00000001:2.0:1775768028.851977:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:0.0:1775768028.851978:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768028.851982:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00020000:00000010:1.0:1775768028.851983:0:8:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994205743180. 00000020:00000040:2.0:1775768028.851986:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00000001:0.0:1775768028.851987:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00008000:00000010:1.0:1775768028.851988:0:8:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994219fec110. 00000080:00000010:1.0:1775768028.851995:0:8:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942057e2960. 00000100:00000001:0.0:1775768028.851996:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1775768028.852000:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000020:00000001:1.0:1775768028.852002:0:8:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107591643720 : -112966117907896 : ffff994206aba248) 00000100:00100000:0.0:1775768028.852008:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfce00 x1862022294265088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00001000:1.0:1775768028.852009:0:8:0:(cl_object.c:831:cl_env_get()) 1@ffff994206aba240 00000100:00000001:2.0:1775768028.852010:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768028.852015:0:8:0:(cl_object.c:151:cl_object_top()) ffff99420791a850 -> ffff99420791a850 00000020:00000001:1.0:1775768028.852021:0:8:0:(cl_io.c:125:__cl_io_init()) Process entered 02000000:00000001:2.0:1775768028.852022:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000080:00000001:1.0:1775768028.852024:0:8:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:1.0:1775768028.852029:0:8:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017f:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000100:00000001:0.0:1775768028.852033:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1775768028.852034:0:8:0:(cl_io.c:898:cl_io_slice_add()) Process entered 02000000:00000001:2.0:1775768028.852034:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1775768028.852042:0:8:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00000040:0.0:1775768028.852045:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfce00 x1862022294265088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768053 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000001:1.0:1775768028.852045:0:8:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768028.852045:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:1.0:1775768028.852055:0:8:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017f:0x0]io ffff994205751e08 type 6 ignore/verify layout 1/0 00000100:00000001:2.0:1775768028.852056:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775768028.852062:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00020000:00000002:1.0:1775768028.852067:0:8:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207dfbd90) owner 0000000000000000/ffff994201224f80 00020000:00000001:1.0:1775768028.852076:0:8:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00000008:00000001:2.0:1775768028.852076:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768028.852077:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00020000:00000001:1.0:1775768028.852080:0:8:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00000100:00000001:0.0:1775768028.852087:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00020000:00000001:1.0:1775768028.852087:0:8:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00000100:00000001:2.0:1775768028.852088:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775768028.852089:0:8:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768028.852096:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00020000:00000001:1.0:1775768028.852098:0:8:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768028.852099:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:2.0:1775768028.852101:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfc700. 00020000:00000001:1.0:1775768028.852104:0:8:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00000100:00000001:0.0:1775768028.852106:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00020000:00000001:1.0:1775768028.852108:0:8:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775768028.852110:0:13471:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00000100:00000001:0.0:1775768028.852111:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:1.0:1775768028.852111:0:8:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:1.0:1775768028.852116:0:8:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:1.0:1775768028.852123:0:8:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 02000000:00000001:2.0:1775768028.852123:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00080000:0.0:1775768028.852124:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00020000:00000002:1.0:1775768028.852125:0:8:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207dfbd90) owner 0000000000000000/ffff994201224f80 02000000:00000001:2.0:1775768028.852129:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768028.852131:0:8:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768028.852135:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768028.852135:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000020:00000001:1.0:1775768028.852136:0:8:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:1.0:1775768028.852142:0:8:0:(cl_io.c:399:cl_io_iter_init()) Process entered 02000000:00000001:2.0:1775768028.852143:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1775768028.852144:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00020000:00000001:1.0:1775768028.852146:0:8:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00020000:00200000:1.0:1775768028.852150:0:8:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 02000000:00000001:2.0:1775768028.852150:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768028.852151:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfce00 x1862022294265088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768053 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000002:1.0:1775768028.852176:0:8:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:1.0:1775768028.852178:0:8:0:(lov_io.c:155:lov_sub_get()) Process entered 00020000:00000001:1.0:1775768028.852179:0:8:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000020:00000001:1.0:1775768028.852180:0:8:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:1.0:1775768028.852183:0:8:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994206ababd0. 00000020:00000010:1.0:1775768028.852190:0:8:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994208a8f600. 02000000:00000010:2.0:1775768028.852190:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1f400. 00000100:00100000:0.0:1775768028.852194:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294265088:192.168.202.156@tcp:38:kworker.0 00000020:00000010:1.0:1775768028.852200:0:8:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994208a8ec00. 00000100:00000001:2.0:1775768028.852202:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768028.852205:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000010:1.0:1775768028.852205:0:8:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207cba9c0. 00000008:00000010:1.0:1775768028.852212:0:8:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207e00000. 00020000:00000010:1.0:1775768028.852214:0:8:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff9942031b30c0. 00000100:00000040:0.0:1775768028.852217:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfed80 x1862022294265216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00008000:00000010:1.0:1775768028.852218:0:8:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942054a8a98. 00000100:00100000:2.0:1775768028.852222:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfc700 x1862022294265472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000080:00000010:1.0:1775768028.852224:0:8:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99422030b0a8. 00000080:00000010:1.0:1775768028.852233:0:8:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff994205753d98. 00000020:00000010:1.0:1775768028.852242:0:8:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994208a8e000. 00000008:00000010:1.0:1775768028.852249:0:8:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff994210709080. 00000100:00000001:0.0:1775768028.852257:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00020000:00000010:1.0:1775768028.852258:0:8:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff9942057418c0. 00000100:00000001:0.0:1775768028.852262:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:1.0:1775768028.852267:0:8:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994219fec368. 02000000:00000001:0.0:1775768028.852272:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000080:00000010:1.0:1775768028.852276:0:8:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942057e24b0. 02000000:00000001:0.0:1775768028.852279:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768028.852279:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000020:00000001:1.0:1775768028.852280:0:8:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107591646168 : -112966117905448 : ffff994206ababd8) 00000100:00000001:2.0:1775768028.852287:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768028.852288:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:1.0:1775768028.852288:0:8:0:(cl_object.c:831:cl_env_get()) 1@ffff994206ababd0 00000020:00000001:1.0:1775768028.852298:0:8:0:(cl_object.c:151:cl_object_top()) ffff99420791a210 -> ffff99420791a850 00000100:00100000:0.0:1775768028.852302:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294265216:192.168.202.156@tcp:38:kworker.0 00000020:00000001:1.0:1775768028.852302:0:8:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:1.0:1775768028.852307:0:8:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000100:00000040:2.0:1775768028.852307:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a176dd1] to pc [ptlrpcd_rcv+-1] req@ffff994207dfc700 x1862022294265472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768028.852308:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000001:1.0:1775768028.852308:0:8:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 02000000:00000001:0.0:1775768028.852312:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:1.0:1775768028.852312:0:8:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768028.852317:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775768028.852318:0:8:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768028.852327:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00020000:00000001:1.0:1775768028.852327:0:8:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107571233256 : -112966138318360 : ffff9942057431e8) 02000000:00000010:0.0:1775768028.852333:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220028400. 00000020:00000001:1.0:1775768028.852337:0:8:0:(cl_io.c:399:cl_io_iter_init()) Process entered 02000000:00000001:0.0:1775768028.852341:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768028.852343:0:8:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775768028.852351:0:8:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768028.852360:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:1.0:1775768028.852361:0:8:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768028.852370:0:8:0:(cl_io.c:314:cl_io_lock()) Process entered 00000400:00000010:0.0:1775768028.852372:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00020000:00000001:1.0:1775768028.852373:0:8:0:(lov_io.c:1108:lov_io_lock()) Process entered 00000100:00000001:2.0:1775768028.852373:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775768028.852375:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00020000:00000001:1.0:1775768028.852377:0:8:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:1.0:1775768028.852383:0:8:0:(cl_io.c:314:cl_io_lock()) Process entered 00000100:00000200:0.0:1775768028.852385:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294265216, portal 10 00000020:00000001:1.0:1775768028.852386:0:8:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:1.0:1775768028.852389:0:8:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768028.852390:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000040:2.0:1775768028.852390:0:13471:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5291) 00000020:00000001:1.0:1775768028.852392:0:8:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768028.852394:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611848064 : -112966097703552 : ffff994207dfed80) 00020000:00000001:1.0:1775768028.852400:0:8:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775768028.852403:0:8:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768028.852406:0:8:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000100:00000040:0.0:1775768028.852408:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfed80 x1862022294265216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768053 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1775768028.852409:0:8:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768028.852413:0:8:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768028.852416:0:8:0:(cl_io.c:528:cl_io_start()) Process entered 00020000:00000001:1.0:1775768028.852419:0:8:0:(lov_io.c:1114:lov_io_start()) Process entered 00000100:00000001:0.0:1775768028.852422:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00020000:00000001:1.0:1775768028.852422:0:8:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:1.0:1775768028.852423:0:8:0:(cl_io.c:528:cl_io_start()) Process entered 00000008:00000001:1.0:1775768028.852425:0:8:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000100:00000040:0.0:1775768028.852426:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000008:00000001:1.0:1775768028.852427:0:8:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000400:00000010:0.0:1775768028.852429:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000008:00000001:1.0:1775768028.852430:0:8:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000100:00000200:0.0:1775768028.852433:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294265216, offset 0 00000008:00000020:1.0:1775768028.852434:0:8:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:1.0:1775768028.852436:0:8:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:1.0:1775768028.852440:0:8:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000008:00000001:1.0:1775768028.852442:0:8:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000400:00000200:0.0:1775768028.852443:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000008:00000001:1.0:1775768028.852445:0:8:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107754938304 : -112965954613312 : ffff994210674fc0) 00000008:00000020:1.0:1775768028.852448:0:8:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994210674fc0 ready 0|-|- wr 1|-|+ rd 0|- 9 in flight 00000400:00000200:0.0:1775768028.852452:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00000001:1.0:1775768028.852454:0:8:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:1.0:1775768028.852456:0:8:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000400:00000200:0.0:1775768028.852458:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000008:00000001:1.0:1775768028.852459:0:8:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:1.0:1775768028.852463:0:8:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000400:00000200:0.0:1775768028.852464:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000008:00000020:1.0:1775768028.852465:0:8:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994218232380 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000400:00000200:0.0:1775768028.852469:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000008:00000001:1.0:1775768028.852470:0:8:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1775768028.852473:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000008:00000001:1.0:1775768028.852474:0:8:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768028.852477:0:8:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768028.852479:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00020000:00000001:1.0:1775768028.852480:0:8:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775768028.852483:0:8:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768028.852486:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225245:225245:256:4294967295] 192.168.202.156@tcp LPNI seq info [225241:225241:8:4294967295] 00000020:00000001:1.0:1775768028.852486:0:8:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768028.852488:0:8:0:(cl_io.c:555:cl_io_end()) Process entered 00020000:00000001:1.0:1775768028.852491:0:8:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00020000:00000001:1.0:1775768028.852493:0:8:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000020:00000001:1.0:1775768028.852495:0:8:0:(cl_io.c:555:cl_io_end()) Process entered 00000400:00000200:0.0:1775768028.852496:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:1.0:1775768028.852496:0:8:0:(cl_io.c:563:cl_io_end()) Process leaving 00020000:00000001:1.0:1775768028.852499:0:8:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775768028.852502:0:8:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000020:00000001:1.0:1775768028.852504:0:8:0:(cl_io.c:563:cl_io_end()) Process leaving 00000020:00000001:1.0:1775768028.852506:0:8:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000400:00000200:0.0:1775768028.852508:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00020000:00000001:1.0:1775768028.852508:0:8:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00020000:00000001:1.0:1775768028.852511:0:8:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:1.0:1775768028.852512:0:8:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000020:00000001:1.0:1775768028.852514:0:8:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:1.0:1775768028.852517:0:8:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1775768028.852520:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00020000:00000001:1.0:1775768028.852520:0:8:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775768028.852523:0:8:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000800:00000010:0.0:1775768028.852524:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000020:00000001:1.0:1775768028.852525:0:8:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:1.0:1775768028.852527:0:8:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000800:00000200:0.0:1775768028.852530:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000001:1.0:1775768028.852530:0:8:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000001:1.0:1775768028.852531:0:8:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:1.0:1775768028.852533:0:8:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00000800:00000200:0.0:1775768028.852534:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00020000:00000001:1.0:1775768028.852535:0:8:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:1.0:1775768028.852536:0:8:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000001:1.0:1775768028.852538:0:8:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00020000:00000001:1.0:1775768028.852540:0:8:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775768028.852542:0:8:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775768028.852543:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00020000:00000001:1.0:1775768028.852544:0:8:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000020:00000001:1.0:1775768028.852546:0:8:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000020:00000001:1.0:1775768028.852548:0:8:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775768028.852548:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000020:00000001:1.0:1775768028.852551:0:8:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:1.0:1775768028.852553:0:8:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:1.0:1775768028.852555:0:8:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000800:00000100:0.0:1775768028.852557:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000020:00000001:1.0:1775768028.852557:0:8:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:1.0:1775768028.852559:0:8:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000800:00000010:0.0:1775768028.852560:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000020:00001000:1.0:1775768028.852561:0:8:0:(cl_object.c:925:cl_env_put()) 1@ffff994206ababd0 00000400:00000200:0.0:1775768028.852565:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000010:1.0:1775768028.852565:0:8:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994208a8ec00. 00000020:00000010:1.0:1775768028.852571:0:8:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207cba9c0. 00000400:00000200:0.0:1775768028.852572:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000008:00000010:1.0:1775768028.852575:0:8:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207e00000. 00000400:00000200:0.0:1775768028.852576:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00020000:00000010:1.0:1775768028.852578:0:8:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff9942031b30c0. 00000400:00000010:0.0:1775768028.852579:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00008000:00000010:1.0:1775768028.852583:0:8:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942054a8a98. 00000100:00000001:0.0:1775768028.852584:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000080:00000010:1.0:1775768028.852588:0:8:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99422030b0a8. 00000100:00000200:0.0:1775768028.852589:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfed80 x1862022294265216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768053 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000010:1.0:1775768028.852591:0:8:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff994205753d98. 00000020:00000010:1.0:1775768028.852595:0:8:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994208a8f600. 00000008:00000010:1.0:1775768028.852600:0:8:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff994210709080. 00000100:00000001:0.0:1775768028.852601:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00020000:00000010:1.0:1775768028.852604:0:8:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff9942057418c0. 00000100:00000040:0.0:1775768028.852605:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfed80 x1862022294265216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768053 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00008000:00000010:1.0:1775768028.852607:0:8:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994219fec368. 00000080:00000010:1.0:1775768028.852611:0:8:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942057e24b0. 00000020:00000010:1.0:1775768028.852626:0:8:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994208a8e000. 00000020:00000010:1.0:1775768028.852630:0:8:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994206ababd0. 00000100:00000001:0.0:1775768028.852631:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775768028.852632:0:8:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00000100:00000001:0.0:1775768028.852634:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00020000:00000001:1.0:1775768028.852635:0:8:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:1.0:1775768028.852637:0:8:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000100:00000001:0.0:1775768028.852638:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000080:00200000:1.0:1775768028.852639:0:8:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017f:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00000040:0.0:1775768028.852644:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000080:00000001:1.0:1775768028.852644:0:8:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:1.0:1775768028.852645:0:8:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:1.0:1775768028.852649:0:8:0:(cl_object.c:925:cl_env_put()) 1@ffff994206aba240 00000400:00000200:0.0:1775768028.852650:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000010:1.0:1775768028.852652:0:8:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994201342e00. 00000100:00000001:0.0:1775768028.852653:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299953677 00000100:00000001:0.0:1775768028.852656:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299955006 00000020:00000010:1.0:1775768028.852656:0:8:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207cba120. 00000100:00000001:0.0:1775768028.852660:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299953677 00000008:00000010:1.0:1775768028.852660:0:8:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207e02ca0. 00000100:00000001:0.0:1775768028.852662:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299955006 00020000:00000010:1.0:1775768028.852664:0:8:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff9942031b3480. 00000100:00000001:0.0:1775768028.852665:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768028.852668:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:1.0:1775768028.852668:0:8:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942054ab170. 00000100:00000001:0.0:1775768028.852672:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000080:00000010:1.0:1775768028.852673:0:8:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99422030bf18. 00000100:00100000:0.0:1775768028.852676:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768028/real 1775768028] req@ffff994207dfed80 x1862022294265216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000010:1.0:1775768028.852676:0:8:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff994205751da8. 00000020:00000010:1.0:1775768028.852678:0:8:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994201343200. 00000008:00000010:1.0:1775768028.852682:0:8:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff994210709ce0. 00020000:00000010:1.0:1775768028.852685:0:8:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994205743180. 00008000:00000010:1.0:1775768028.852689:0:8:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994219fec110. 00000400:00000001:0.0:1775768028.852694:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000080:00000010:1.0:1775768028.852695:0:8:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942057e2960. 00000400:00000001:0.0:1775768028.852697:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000010:1.0:1775768028.852698:0:8:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994201343000. 00000400:00000010:0.0:1775768028.852699:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775768028.852704:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000020:00000010:1.0:1775768028.852704:0:8:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994206aba240. 00000400:00000010:0.0:1775768028.852708:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000080:00000001:1.0:1775768028.852708:0:8:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768028.852711:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000080:00000001:1.0:1775768028.852711:0:8:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1775768028.852716:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfed80 x1862022294265216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768028.852723:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfed80 x1862022294265216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768028.852727:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768028.852728:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768028.852729:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768028.852730:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768028.852733:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfed80 x1862022294265216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768028.852737:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768028.852740:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfed80 x1862022294265216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768053 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768028.852744:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768028.852747:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768028.852748:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768028.852749:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768028.852750:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768028.852752:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768028.852754:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768028.852757:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768028.852758:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768028.852760:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfed80 x1862022294265216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768053 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768028.852765:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294265216:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768028.852767:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768028.852769:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768028.852770:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfce00 x1862022294265088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768053 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768028.852774:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768028.852775:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768028.852777:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220028000. 02000000:00000001:0.0:1775768028.852779:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768028.852780:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768028.852781:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:0.0:1775768028.852782:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768028.852784:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1d000. 02000000:00000001:0.0:1775768028.852787:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768028.852788:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768028.852789:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfce00. 00000100:00000001:0.0:1775768028.852791:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768028.852792:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768028.852793:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768028.852795:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfed80 x1862022294265216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768053 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768028.852804:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768028.852805:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768028.852806:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220028400. 02000000:00000001:0.0:1775768028.852807:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768028.852808:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768028.852808:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:0.0:1775768028.852810:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768028.852810:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1f800. 02000000:00000001:0.0:1775768028.852811:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768028.852812:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768028.852813:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfed80. 00000100:00000001:0.0:1775768028.852814:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768028.852815:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768028.852816:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768028.852818:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768028.852819:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768028.852820:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775768028.852822:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768028.852823:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768028.852825:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfe300 x1862022294265344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768028.852829:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768028.852830:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768028.852832:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768028.852833:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768028.852834:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768028.852836:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294265344:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768028.852838:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768028.852839:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768028.852841:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768028.852842:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768028.852844:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220028400. 02000000:00000001:0.0:1775768028.852845:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768028.852848:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768028.852850:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775768028.852852:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768028.852855:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294265344, portal 4 00000100:00000001:0.0:1775768028.852856:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768028.852857:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611845376 : -112966097706240 : ffff994207dfe300) 00000100:00000040:0.0:1775768028.852860:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfe300 x1862022294265344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768053 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768028.852865:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768028.852866:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768028.852868:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775768028.852869:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294265344, offset 0 00000400:00000200:0.0:1775768028.852872:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768028.852874:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768028.852876:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768028.852878:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768028.852879:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768028.852881:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768028.852883:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768028.852885:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225246:225246:256:4294967295] 192.168.202.156@tcp LPNI seq info [225242:225242:8:4294967295] 00000400:00000200:0.0:1775768028.852888:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768028.852892:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768028.852895:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768028.852897:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775768028.852900:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768028.852902:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775768028.852905:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768028.852907:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1775768028.852910:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768028.852911:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775768028.852913:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768028.852915:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768028.852917:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775768028.852919:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775768028.852921:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768028.852923:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfe300 x1862022294265344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768053 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768028.852928:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768028.852930:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfe300 x1862022294265344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768053 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768028.852934:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768028.852935:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768028.852937:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768028.852939:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:0.0:1775768028.852941:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768028.852943:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299953700 00000100:00000001:0.0:1775768028.852944:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299955006 00000100:00000001:0.0:1775768028.852946:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299953700 00000100:00000001:0.0:1775768028.852948:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299955006 00000100:00000001:0.0:1775768028.852949:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768028.852950:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768028.852951:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768028.852953:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768028/real 1775768028] req@ffff994207dfe300 x1862022294265344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768028.852958:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768028.852959:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768028.852960:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775768028.852961:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768028.852962:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768028.852963:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768028.852965:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfe300 x1862022294265344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768028.852970:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfe300 x1862022294265344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768028.852975:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768028.852976:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768028.852978:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768028.852979:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768028.852981:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfe300 x1862022294265344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768028.852985:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768028.852987:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfe300 x1862022294265344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768053 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768028.852991:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768028.852992:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768028.852993:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768028.852994:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768028.852995:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768028.852998:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768028.853000:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768028.853002:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768028.853003:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768028.853005:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfe300 x1862022294265344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768053 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768028.853010:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294265344:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768028.853012:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768028.853014:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfc700 x1862022294265472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768028.853019:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768028.853020:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768028.853022:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768028.853023:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768028.853024:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768028.853026:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294265472:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768028.853029:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768028.853030:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768028.853030:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768028.853031:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768028.853033:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220028000. 02000000:00000001:0.0:1775768028.853034:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768028.853036:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768028.853037:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775768028.853039:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768028.853041:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294265472, portal 4 00000100:00000001:0.0:1775768028.853043:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768028.853044:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611838208 : -112966097713408 : ffff994207dfc700) 00000100:00000040:0.0:1775768028.853047:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfc700 x1862022294265472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768053 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768028.853051:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768028.853052:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768028.853054:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775768028.853055:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294265472, offset 0 00000400:00000200:0.0:1775768028.853057:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768028.853060:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768028.853062:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768028.853064:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768028.853065:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768028.853067:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768028.853068:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768028.853071:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225247:225247:256:4294967295] 192.168.202.156@tcp LPNI seq info [225243:225243:8:4294967295] 00000400:00000200:0.0:1775768028.853073:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768028.853077:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768028.853080:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768028.853082:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775768028.853085:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768028.853087:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775768028.853090:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768028.853092:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1775768028.853095:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768028.853096:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775768028.853098:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768028.853101:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768028.853102:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775768028.853103:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775768028.853104:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768028.853106:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfc700 x1862022294265472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768053 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768028.853112:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768028.853114:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfc700 x1862022294265472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768053 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768028.853118:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768028.853119:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768028.853121:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768028.853123:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:0.0:1775768028.853124:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768028.853125:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299953703 00000100:00000001:0.0:1775768028.853127:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299955006 00000100:00000001:0.0:1775768028.853129:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299953703 00000100:00000001:0.0:1775768028.853131:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299955006 00000100:00000001:0.0:1775768028.853132:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768028.853133:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768028.853134:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768028.853136:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768028/real 1775768028] req@ffff994207dfc700 x1862022294265472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768028.853142:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768028.853143:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768028.853144:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775768028.853145:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768028.853146:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768028.853148:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768028.853149:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfc700 x1862022294265472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768028.853171:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfc700 x1862022294265472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768028.853176:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768028.853177:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768028.853178:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768028.853178:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768028.853181:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfc700 x1862022294265472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768028.853185:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768028.853187:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfc700 x1862022294265472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768053 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768028.853192:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768028.853193:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768028.853194:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768028.853195:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768028.853196:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768028.853199:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768028.853201:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768028.853203:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768028.853204:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768028.853207:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfc700 x1862022294265472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768053 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768028.853212:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294265472:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768028.853215:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768028.853216:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768028.853218:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfe300 x1862022294265344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768053 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768028.853221:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768028.853222:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768028.853223:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220028400. 02000000:00000001:0.0:1775768028.853225:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768028.853225:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768028.853226:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:0.0:1775768028.853227:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768028.853228:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1e400. 02000000:00000001:0.0:1775768028.853231:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768028.853232:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768028.853233:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfe300. 00000100:00000001:0.0:1775768028.853234:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768028.853235:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768028.853236:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768028.853238:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfc700 x1862022294265472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768053 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768028.853242:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768028.853243:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768028.853244:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220028000. 02000000:00000001:0.0:1775768028.853245:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768028.853245:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768028.853246:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:0.0:1775768028.853247:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768028.853248:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1f400. 02000000:00000001:0.0:1775768028.853249:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768028.853250:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768028.853251:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfc700. 00000100:00000001:0.0:1775768028.853253:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768028.853254:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768028.853255:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768028.853257:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768028.853258:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768028.853259:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775768028.853260:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768028.913299:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768028.913308:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768028.913316:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768028.913317:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768028.913322:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768028.913324:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768028.913328:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768028.913330:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768028.913334:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768028.913335:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768028.913362:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768028.913365:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768028.913369:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768028.977304:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768028.977312:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768028.977314:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768028.977320:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768028.977324:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768028.977328:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768028.977331:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768028.977333:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768028.977336:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768029.873275:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768029.873275:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768029.873275:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768029.873285:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768029.873285:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768029.873289:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768029.873290:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768029.873292:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768029.873295:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768029.873299:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775768029.873299:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:0.0:1775768029.873300:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775768029.873307:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768029.873307:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768029.873307:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768029.873314:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768029.873315:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768029.873315:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768029.873322:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768029.873325:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768029.873327:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768029.873330:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768029.873334:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768029.873337:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768029.873338:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768029.873342:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768029.873345:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768029.937244:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768029.937248:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768029.937253:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768029.937254:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768029.937257:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768029.937258:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768029.937260:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768029.937261:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768029.937263:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768029.937264:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768029.937279:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768029.937281:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768029.937283:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768030.001245:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768030.001252:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768030.001255:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768030.001260:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768030.001264:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768030.001269:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768030.001271:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768030.001274:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768030.001277:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768030.193275:0:14953:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4299955006, at 4299955008 00000100:00000040:1.0:1775768030.193275:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299955006, at 4299955008 00000100:00000010:0.0:1775768030.193293:0:14953:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3d80. 00000100:00000010:1.0:1775768030.193296:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3c00. 00000100:00000001:3.0:1775768030.897255:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768030.897256:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768030.897257:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768030.897262:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768030.897263:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768030.897263:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768030.897264:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768030.897266:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768030.897267:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768030.897272:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775768030.897272:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:0.0:1775768030.897273:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768030.897278:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768030.897279:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768030.897280:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768030.897282:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768030.897284:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768030.897284:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768030.897285:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768030.897288:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768030.897291:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768030.897292:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768030.897295:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768030.897299:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768030.897299:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768030.897303:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768030.897306:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768030.961254:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768030.961258:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768030.961263:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768030.961264:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768030.961267:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768030.961268:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768030.961271:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768030.961272:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768030.961274:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768030.961274:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768030.961290:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768030.961292:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768030.961294:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768031.025258:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768031.025264:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768031.025265:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768031.025270:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768031.025273:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768031.025276:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768031.025278:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768031.025280:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768031.025282:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768031.921267:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768031.921267:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768031.921274:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768031.921274:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768031.921277:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768031.921277:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768031.921283:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775768031.921283:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768031.921288:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768031.921288:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768031.921293:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768031.921293:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768031.921295:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768031.921296:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768031.921300:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768031.921301:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768031.921302:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768031.921307:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768031.921307:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768031.921307:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768031.921310:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768031.921310:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768031.921314:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768031.921314:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768031.921317:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768031.921319:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768031.921321:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768031.985245:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768031.985251:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768031.985254:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768031.985257:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768031.985262:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768031.985264:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768031.985267:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768031.985269:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768031.985273:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768031.985275:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768031.985278:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768031.985279:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768031.985283:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768031.985284:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768031.985287:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768031.985289:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768031.985307:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768031.985310:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768031.985314:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768032.049242:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768032.049249:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768032.049251:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768032.049257:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768032.049261:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768032.049265:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768032.049268:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768032.049271:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768032.049274:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768032.945259:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768032.945265:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768032.945267:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768032.945271:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:3.0:1775768032.945272:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768032.945277:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768032.945278:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768032.945281:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768032.945282:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768032.945285:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775768032.945286:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768032.945290:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768032.945291:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768032.945293:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768032.945295:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768032.945295:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768032.945301:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768032.945302:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768032.945303:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768032.945305:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768032.945308:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768032.945310:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768032.945316:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768032.945319:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768032.945322:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768032.945324:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768032.945327:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768033.009218:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768033.009222:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768033.009227:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768033.009228:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768033.009231:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768033.009232:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768033.009234:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768033.009234:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768033.009236:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768033.009237:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768033.009253:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768033.009255:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768033.009257:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768033.073231:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768033.073236:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768033.073238:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768033.073242:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768033.073245:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768033.073247:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768033.073250:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768033.073252:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768033.073254:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768033.969267:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768033.969275:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768033.969276:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768033.969276:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768033.969278:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768033.969279:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768033.969282:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768033.969283:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:3.0:1775768033.969284:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:1.0:1775768033.969285:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00080000:0.0:1775768033.969286:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775768033.969287:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768033.969289:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768033.969289:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:2.0:1775768033.969293:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00080000:0.0:1775768033.969294:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775768033.969295:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768033.969295:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768033.969299:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768033.969300:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775768033.969300:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1775768033.969304:0:8:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775768033.969304:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768033.969305:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768033.969309:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775768033.969312:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768033.969312:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768033.969315:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1775768033.969317:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5286 00000100:00000001:3.0:1775768033.969318:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768033.969319:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768033.969323:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1775768033.969327:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768033.969329:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768033.969334:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768033.969337:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768033.969340:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768033.969343:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768033.969346:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768033.969348:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768033.969351:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768033.969353:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:0.0:1775768033.969356:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768033.969358:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:0.0:1775768033.969359:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:0.0:1775768033.969361:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:0.0:1775768033.969363:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768033.969365:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768033.969366:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768033.969368:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768033.969371:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:0.0:1775768033.969373:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768033.969375:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768033.969377:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768033.969379:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768033.969381:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768033.969383:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775768033.969384:0:8:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768033.969392:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fc000. 00000020:00000040:0.0:1775768033.969394:0:8:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:0.0:1775768033.969397:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768033.969398:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768033.969400:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768033.969403:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768033.969407:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768033.969423:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220028400. 00000100:00000001:0.0:1775768033.969429:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768033.969436:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fc000 x1862022294265600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768033.969444:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768033.969448:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768033.969451:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008bcb9792] to pc [ptlrpcd_rcv+-1] req@ffff9942099fc000 x1862022294265600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768033.969470:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768033.969474:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768033.969478:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775768033.969480:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768033.969482:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768033.969485:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768033.969489:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5286 00000100:00000001:1.0:1775768033.969492:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768033.969495:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000400:00000200:0.0:1775768033.969496:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768033.969498:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000400:00000200:0.0:1775768033.969504:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000040:1.0:1775768033.969507:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fc000 x1862022294265600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768033.969508:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768033.969517:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768033.969518:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768033.969521:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768033.969522:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1775768033.969524:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:0.0:1775768033.969527:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000001:1.0:1775768033.969527:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768033.969532:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768033.969533:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00100000:1.0:1775768033.969536:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294265600:192.168.202.156@tcp:250:kworker.0 00000020:00000001:0.0:1775768033.969538:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768033.969541:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000100:00000001:1.0:1775768033.969542:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000001:0.0:1775768033.969545:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000001:1.0:1775768033.969547:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768033.969549:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768033.969551:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768033.969556:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543d000. 00000020:00000040:0.0:1775768033.969559:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 02000000:00000001:1.0:1775768033.969559:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768033.969564:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000400:00000200:1.0:1775768033.969586:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:0.0:1775768033.969590:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000400:00000010:1.0:1775768033.969593:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023cc08. 00000100:00000040:0.0:1775768033.969594:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768033.969598:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768033.969600:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768033.969602:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000010:1.0:1775768033.969602:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf67f8. 00000020:00000040:0.0:1775768033.969607:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00080000:0.0:1775768033.969610:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768033.969611:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775768033.969612:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294265600, portal 25 02000000:00000001:0.0:1775768033.969613:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1775768033.969617:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 02000000:00000001:0.0:1775768033.969618:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768033.969619:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641196544 : -112966068355072 : ffff9942099fc000) 02000000:00000001:0.0:1775768033.969622:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768033.969624:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775768033.969628:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00000040:1.0:1775768033.969630:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fc000 x1862022294265600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768058 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:0.0:1775768033.969631:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768033.969634:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768033.969639:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ff800. 00000020:00000040:0.0:1775768033.969643:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:0.0:1775768033.969648:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1775768033.969649:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:0.0:1775768033.969650:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768033.969653:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000040:1.0:1775768033.969653:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 02000000:00000001:0.0:1775768033.969654:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768033.969657:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775768033.969658:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 02000000:00000010:0.0:1775768033.969661:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220028000. 00000100:00000200:1.0:1775768033.969662:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294265600, offset 0 00000100:00000001:0.0:1775768033.969667:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768033.969670:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00100000:0.0:1775768033.969674:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ff800 x1862022294265728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775768033.969681:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768033.969683:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768033.969686:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768033.969688:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00000040:0.0:1775768033.969693:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000aa4e8b71] to pc [ptlrpcd_rcv+-1] req@ffff9942099ff800 x1862022294265728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775768033.969696:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768033.969703:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000100:00000001:0.0:1775768033.969704:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768033.969708:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000100:00080000:0.0:1775768033.969709:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1775768033.969712:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00080000:0.0:1775768033.969714:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775768033.969716:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768033.969719:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1775768033.969719:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225248:225248:256:4294967295] 192.168.202.156@tcp LPNI seq info [225244:225244:8:4294967295] 00000100:00000001:0.0:1775768033.969725:0:8:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:1.0:1775768033.969728:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00080000:0.0:1775768033.969729:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5286 00000400:00000200:1.0:1775768033.969738:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768033.969744:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768033.969745:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000800:00000010:1.0:1775768033.969750:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541200. 00000400:00000200:0.0:1775768033.969751:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768033.969754:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768033.969757:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768033.969760:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:1.0:1775768033.969761:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000040:0.0:1775768033.969765:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768033.969767:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000800:00000200:1.1:1775768033.969770:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000001:0.0:1775768033.969771:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768033.969773:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:0.0:1775768033.969776:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768033.969778:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:0.0:1775768033.969779:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:0.0:1775768033.969781:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:0.0:1775768033.969783:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768033.969785:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775768033.969786:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768033.969787:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768033.969791:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768033.969794:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000400:00000200:1.0:1775768033.969795:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00080000:0.0:1775768033.969798:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775768033.969799:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299960125 00000100:00000001:0.0:1775768033.969803:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768033.969806:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000010:1.0:1775768033.969806:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba120. 02000000:00000001:0.0:1775768033.969808:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768033.969811:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768033.969812:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775768033.969813:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000008:00100000:0.0:1775768033.969814:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775768033.969818:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768033.969821:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768033.969822:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299960125 00000100:00000010:0.0:1775768033.969824:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fd500. 00000100:00000010:1.0:1775768033.969826:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba9c0. 00000020:00000040:0.0:1775768033.969827:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:0.0:1775768033.969830:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1775768033.969830:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 02000000:00000001:0.0:1775768033.969833:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768033.969834:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768033.969835:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768033.969838:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1775768033.969839:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768033.969840:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768033.969842:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768033.969845:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002a800. 00000100:00000001:1.0:1775768033.969845:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768033.969850:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768033.969854:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:0.0:1775768033.969855:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fd500 x1862022294265856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775768033.969857:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1775768033.969861:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768033.969864:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768033.969865:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775768033.969867:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1775768033.969868:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768033.969873:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ff800 x1862022294265728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775768033.969874:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f246bb2b] to pc [ptlrpcd_rcv+-1] req@ffff9942099fd500 x1862022294265856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768033.969900:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768033.969902:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768033.969903:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768033.969905:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1775768033.969906:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:0.0:1775768033.969909:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 02000000:00000001:1.0:1775768033.969910:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768033.969911:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:1.0:1775768033.969913:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768033.969914:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768033.969918:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00100000:1.0:1775768033.969920:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294265728:192.168.202.156@tcp:38:kworker.0 00000100:00080000:0.0:1775768033.969923:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5286 00000100:00000001:1.0:1775768033.969926:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000400:00000200:0.0:1775768033.969929:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:1.0:1775768033.969929:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768033.969932:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768033.969933:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 02000000:00000001:1.0:1775768033.969935:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:0.0:1775768033.969936:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 02000000:00000010:1.0:1775768033.969939:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543d400. 00000100:00000040:0.0:1775768033.969940:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768033.969944:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1775768033.969944:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768033.969948:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000200:1.0:1775768033.969951:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768033.969952:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768033.969955:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000400:00000010:1.0:1775768033.969955:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c318. 00000020:00000040:0.0:1775768033.969958:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000400:00000010:1.0:1775768033.969960:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6440. 00000020:00000001:0.0:1775768033.969961:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768033.969964:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000100:00000200:1.0:1775768033.969965:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294265728, portal 10 00000020:00000001:0.0:1775768033.969967:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:1.0:1775768033.969968:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768033.969971:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641210880 : -112966068340736 : ffff9942099ff800) 00000020:00000001:0.0:1775768033.969972:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:0.0:1775768033.969978:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768033.969980:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768033.969982:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ff800 x1862022294265728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768058 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775768033.969985:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768033.969989:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768033.969993:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00000001:1.0:1775768033.969997:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1775768033.969999:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000040:1.0:1775768033.970003:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768033.970005:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf62a8. 00000100:00000200:1.0:1775768033.970007:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294265728, offset 0 00000400:00000200:1.0:1775768033.970009:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00000001:0.0:1775768033.970013:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768033.970013:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768033.970017:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 02000000:00000001:0.0:1775768033.970018:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1775768033.970021:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:0.0:1775768033.970023:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1775768033.970025:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 02000000:00000001:0.0:1775768033.970029:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768033.970029:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1775768033.970035:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:1.0:1775768033.970035:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000008:00100000:0.0:1775768033.970039:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000400:00000200:1.0:1775768033.970040:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225249:225249:255:4294967295] 192.168.202.156@tcp LPNI seq info [225245:225245:7:4294967295] 00000008:00000001:0.0:1775768033.970046:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768033.970051:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768033.970051:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000010:0.0:1775768033.970058:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fc700. 00000020:00000040:0.0:1775768033.970063:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00000400:00000200:1.0:1775768033.970064:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 02000000:00000001:0.0:1775768033.970069:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768033.970072:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775768033.970074:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00000001:0.0:1775768033.970076:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768033.970079:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768033.970081:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1775768033.970081:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541700. 00000800:00000200:1.0:1775768033.970090:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 02000000:00000010:0.0:1775768033.970091:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002bc00. 00000100:00000001:0.0:1775768033.970097:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775768033.970098:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:1.0:1775768033.970103:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768033.970107:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fc700 x1862022294265984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:1.0:1775768033.970108:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:1.0:1775768033.970115:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768033.970120:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299958802 00000100:00000001:1.0:1775768033.970126:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299960125 00000100:00000001:0.0:1775768033.970127:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768033.970132:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768033.970132:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299958802 00000100:00000001:1.0:1775768033.970138:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299960125 00000100:00000040:0.0:1775768033.970142:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003283b20a] to pc [ptlrpcd_rcv+-1] req@ffff9942099fc700 x1862022294265984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768033.970143:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768033.970147:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768033.970152:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768033.970179:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768033.970183:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768033.970185:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1775768033.970187:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768033.970189:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768033.970191:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768033.970196:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fd500 x1862022294265856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768033.970215:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768033.970216:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768033.970219:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768033.970222:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1775768033.970223:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768033.970226:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:0.0:1775768033.970230:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 02000000:00000001:1.0:1775768033.970231:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768033.970234:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768033.970237:0:8:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775768033.970238:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294265856:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768033.970244:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775768033.970245:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768033.970249:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:0.0:1775768033.970250:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5286 02000000:00000001:1.0:1775768033.970251:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768033.970255:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:0.0:1775768033.970259:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000010:1.0:1775768033.970260:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543e400. 00000400:00000200:0.0:1775768033.970265:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 02000000:00000001:1.0:1775768033.970266:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768033.970271:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1775768033.970274:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000040:0.0:1775768033.970279:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000010:1.0:1775768033.970279:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c6e0. 00000100:00000001:0.0:1775768033.970283:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:1.0:1775768033.970286:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf63b8. 00000100:00000040:0.0:1775768033.970287:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768033.970291:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000200:1.0:1775768033.970292:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294265856, portal 10 00000020:00000001:0.0:1775768033.970296:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:1.0:1775768033.970297:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000020:00000040:0.0:1775768033.970299:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000100:00000001:1.0:1775768033.970299:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641201920 : -112966068349696 : ffff9942099fd500) 00000020:00000001:0.0:1775768033.970302:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768033.970305:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:0.0:1775768033.970308:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:1.0:1775768033.970311:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fd500 x1862022294265856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768058 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775768033.970312:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:0.0:1775768033.970319:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768033.970324:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768033.970328:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768033.970329:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1775768033.970333:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000040:1.0:1775768033.970336:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000020:00000040:0.0:1775768033.970338:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000400:00000010:1.0:1775768033.970340:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6990. 00000100:00080000:0.0:1775768033.970344:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000200:1.0:1775768033.970345:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294265856, offset 0 00000100:00000001:0.0:1775768033.970349:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768033.970354:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1775768033.970356:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 02000000:00000001:0.0:1775768033.970359:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768033.970364:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768033.970364:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768033.970370:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:1.0:1775768033.970372:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000008:00100000:0.0:1775768033.970374:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:0.0:1775768033.970378:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768033.970379:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1775768033.970383:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768033.970384:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768033.970390:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000100:00000010:0.0:1775768033.970391:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ff100. 00000400:00000200:1.0:1775768033.970393:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000020:00000040:0.0:1775768033.970396:0:8:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00000400:00000200:1.0:1775768033.970401:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225250:225250:254:4294967295] 192.168.202.156@tcp LPNI seq info [225246:225246:6:4294967295] 02000000:00000001:0.0:1775768033.970402:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768033.970405:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768033.970410:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:1.0:1775768033.970410:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:0.0:1775768033.970413:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768033.970417:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768033.970423:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 02000000:00000010:0.0:1775768033.970424:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002b800. 00000100:00000001:0.0:1775768033.970431:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775768033.970434:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00100000:0.0:1775768033.970439:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ff100 x1862022294266112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000010:1.0:1775768033.970440:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541100. 00000800:00000200:1.0:1775768033.970452:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:0.0:1775768033.970455:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768033.970460:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775768033.970460:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:1.0:1775768033.970468:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768033.970469:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5e24158] to pc [ptlrpcd_rcv+-1] req@ffff9942099ff100 x1862022294266112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775768033.970474:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:1.0:1775768033.970483:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768033.970487:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299958795 00000100:00000001:0.0:1775768033.970488:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768033.970492:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299960125 00000100:00000040:0.0:1775768033.970495:0:8:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5296) 00000100:00000001:1.0:1775768033.970496:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299958795 00000100:00000001:1.0:1775768033.970501:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299960125 00000100:00000001:1.0:1775768033.970503:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768033.970506:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768033.970509:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768033.970511:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768033.970515:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768033.970516:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1775768033.970519:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768033.970521:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768033.970522:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768033.970526:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fc700 x1862022294265984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768033.970533:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768033.970535:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768033.970537:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768033.970539:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768033.970541:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768033.970543:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294265984:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768033.970546:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768033.970548:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768033.970549:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768033.970551:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768033.970555:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543d800. 02000000:00000001:1.0:1775768033.970556:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768033.970559:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768033.970562:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023cdc0. 00000400:00000010:1.0:1775768033.970583:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6d48. 00000100:00000200:1.0:1775768033.970587:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294265984, portal 4 00000100:00000001:1.0:1775768033.970588:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768033.970590:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641198336 : -112966068353280 : ffff9942099fc700) 00000100:00000040:1.0:1775768033.970595:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fc700 x1862022294265984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768058 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768033.970602:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768033.970603:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768033.970607:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768033.970609:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294265984, offset 0 00000400:00000200:1.0:1775768033.970614:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768033.970618:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768033.970621:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768033.970624:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768033.970627:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768033.970631:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768033.970634:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768033.970637:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225251:225251:253:4294967295] 192.168.202.156@tcp LPNI seq info [225247:225247:5:4294967295] 00000400:00000200:1.0:1775768033.970642:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768033.970648:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768033.970653:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768033.970658:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541600. 00000800:00000200:1.0:1775768033.970662:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768033.970666:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:1.0:1775768033.970668:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768033.970672:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:1.0:1775768033.970675:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768033.970677:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299958818 00000100:00000001:1.0:1775768033.970680:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299960125 00000100:00000001:1.0:1775768033.970682:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299958818 00000100:00000001:1.0:1775768033.970684:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299960125 00000100:00000001:1.0:1775768033.970687:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768033.970689:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768033.970692:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768033.970696:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ff100 x1862022294266112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768033.970702:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768033.970704:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768033.970706:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768033.970708:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768033.970710:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768033.970713:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294266112:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768033.970717:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768033.970719:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768033.970720:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768033.970722:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768033.970726:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543c400. 02000000:00000001:1.0:1775768033.970728:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768033.970730:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768033.970734:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023cf78. 00000400:00000010:1.0:1775768033.970737:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6aa0. 00000100:00000200:1.0:1775768033.970740:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294266112, portal 4 00000100:00000001:1.0:1775768033.970742:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768033.970743:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641209088 : -112966068342528 : ffff9942099ff100) 00000100:00000040:1.0:1775768033.970749:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ff100 x1862022294266112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768058 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768033.970755:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768033.970757:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768033.970759:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000200:1.0:1775768033.970761:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294266112, offset 0 00000400:00000200:1.0:1775768033.970765:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768033.970769:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768033.970772:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768033.970776:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768033.970778:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768033.970780:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768033.970782:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768033.970785:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225252:225252:252:4294967295] 192.168.202.156@tcp LPNI seq info [225248:225248:4:4294967295] 00000400:00000200:1.0:1775768033.970789:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768033.970795:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768033.970799:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768033.970802:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000800:00000200:1.0:1775768033.970806:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768033.970809:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:1.0:1775768033.970811:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768033.970813:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:1.0:1775768033.970816:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768033.970819:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299958821 00000100:00000001:1.0:1775768033.970821:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299960125 00000100:00000001:1.0:1775768033.970824:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299958821 00000100:00000001:1.0:1775768033.970826:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299960125 00000100:00000001:1.0:1775768033.970829:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768033.970831:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768033.970833:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768033.970835:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768033.970838:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768033.970840:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1775768033.970842:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768033.970843:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768033.970845:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768033.970847:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768034.033223:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768034.033227:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768034.033231:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768034.033232:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768034.033234:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768034.033235:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768034.033238:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768034.033239:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768034.033241:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768034.033242:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768034.033257:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768034.033259:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768034.033261:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768034.097236:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768034.097243:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768034.097245:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768034.097250:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768034.097254:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768034.097257:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768034.097259:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768034.097261:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768034.097263:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768034.993248:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768034.993248:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768034.993258:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768034.993259:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768034.993262:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768034.993262:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768034.993270:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775768034.993270:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768034.993276:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768034.993277:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768034.993282:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768034.993282:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768034.993287:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768034.993291:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768034.993294:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768034.993295:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768034.993297:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768034.993300:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768034.993373:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768034.993378:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768034.993379:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768034.993384:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775768034.993386:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768034.993389:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768034.993390:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768034.993392:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768034.993394:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768035.057313:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768035.057321:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768035.057326:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768035.057327:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768035.057331:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768035.057333:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768035.057336:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768035.057338:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768035.057342:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768035.057343:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768035.057369:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768035.057372:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768035.057376:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768035.121312:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768035.121319:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768035.121320:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768035.121326:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768035.121331:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768035.121335:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768035.121337:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768035.121340:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768035.121342:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768035.313265:0:14953:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4299960125, at 4299960128 00000100:00000040:1.0:1775768035.313265:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299960125, at 4299960128 00000100:00000010:0.0:1775768035.313282:0:14953:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cba120. 00000100:00000010:1.0:1775768035.313287:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cba9c0. 00000100:00000001:3.0:1775768036.017279:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768036.017287:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768036.017289:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768036.017290:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768036.017290:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768036.017298:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768036.017300:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768036.017301:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768036.017301:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:0.0:1775768036.017303:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768036.017314:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:0.0:1775768036.017315:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768036.017316:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768036.017324:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768036.017325:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768036.017331:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768036.017331:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768036.017333:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768036.017339:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768036.017343:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768036.017346:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768036.017347:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768036.017350:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768036.017353:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768036.017354:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768036.017358:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768036.017362:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768036.081290:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768036.081297:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768036.081302:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768036.081305:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768036.081307:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768036.081310:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768036.081313:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768036.081315:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768036.081317:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768036.081318:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768036.081322:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768036.081323:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768036.081324:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768036.081325:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768036.081325:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768036.081326:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768036.081327:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768036.081329:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768036.081331:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768036.081332:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768036.081333:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768036.081335:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768036.081336:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768036.081337:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768036.081338:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768036.081340:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768036.081343:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768036.081344:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768036.081344:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768036.081345:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768036.081346:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768036.081347:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768036.081348:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768036.081349:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768036.081371:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768036.081374:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768036.081376:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768036.145267:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768036.145276:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768036.145278:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768036.145284:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768036.145288:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768036.145292:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768036.145295:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768036.145297:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768036.145300:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768037.041223:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768037.041224:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768037.041230:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768037.041230:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768037.041232:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768037.041233:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768037.041239:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775768037.041239:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768037.041245:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768037.041245:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768037.041249:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768037.041250:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768037.041253:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768037.041256:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768037.041258:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768037.041262:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768037.041266:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768037.041269:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768037.041271:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768037.041276:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768037.041278:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768037.041282:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775768037.041286:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768037.041289:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768037.041291:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768037.041293:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768037.041296:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:1.0:1775768037.041445:0:2387:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.156:988 00000400:00000100:1.0:1775768037.041485:0:2387:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.156@tcp at host 192.168.202.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:1.0:1775768037.041496:0:2387:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768037.041500:0:2387:0:(router.c:1750:lnet_notify()) 192.168.202.56@tcp notifying 192.168.202.156@tcp: down 00000400:00000200:1.0:1775768037.041505:0:2387:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768037.041508:0:2387:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:1.0:1775768037.041514:0:2387:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000800:00000010:1.0:1775768037.041518:0:2387:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541200. 00000400:00000200:1.0:1775768037.041523:0:2387:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775768037.041528:0:2387:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775768037.041562:0:2387:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000400:00000010:1.0:1775768037.041566:0:2387:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000100:00000001:1.0:1775768037.041572:0:2387:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768037.041580:0:2387:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9942099fc000 x1862022294265600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768058 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768037.041608:0:2387:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768037.041612:0:2387:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fc000 x1862022294265600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768058 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768037.041620:0:2387:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768037.041622:0:2387:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775768037.041631:0:2387:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000800:00000010:1.0:1775768037.041633:0:2387:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541700. 00000400:00000200:1.0:1775768037.041637:0:2387:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775768037.041640:0:2387:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775768037.041644:0:2387:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf62a8 00000400:00000010:1.0:1775768037.041646:0:2387:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf62a8. 00000100:00000001:1.0:1775768037.041648:0:2387:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768037.041653:0:2387:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9942099ff800 x1862022294265728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768058 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768037.041661:0:2387:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768037.041664:0:2387:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ff800 x1862022294265728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768058 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768037.041671:0:2387:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768037.041672:0:2387:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775768037.041677:0:2387:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000800:00000010:1.0:1775768037.041679:0:2387:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541100. 00000400:00000200:1.0:1775768037.041682:0:2387:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775768037.041685:0:2387:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775768037.041689:0:2387:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6990 00000400:00000010:1.0:1775768037.041690:0:2387:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6990. 00000100:00000001:1.0:1775768037.041693:0:2387:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768037.041697:0:2387:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9942099fd500 x1862022294265856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768058 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768037.041704:0:2387:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768037.041707:0:2387:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fd500 x1862022294265856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768058 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768037.041713:0:2387:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768037.041715:0:2387:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775768037.041719:0:2387:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000800:00000010:1.0:1775768037.041721:0:2387:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541600. 00000400:00000200:1.0:1775768037.041724:0:2387:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775768037.041728:0:2387:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775768037.041730:0:2387:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768037.041732:0:2387:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768037.041735:0:2387:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768037.041739:0:2387:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9942099fc700 x1862022294265984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768058 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768037.041747:0:2387:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768037.041750:0:2387:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fc700 x1862022294265984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768058 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768037.041756:0:2387:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768037.041757:0:2387:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775768037.041762:0:2387:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000800:00000010:1.0:1775768037.041765:0:2387:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768037.041767:0:2387:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775768037.041770:0:2387:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775768037.041773:0:2387:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768037.041775:0:2387:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768037.041778:0:2387:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768037.041783:0:2387:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9942099ff100 x1862022294266112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768058 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768037.041798:0:2387:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768037.041801:0:2387:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ff100 x1862022294266112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768058 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768037.041807:0:2387:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768037.041808:0:2387:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768037.042241:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775768037.042245:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768037.042248:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768037.042254:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768033/real 1775768037] req@ffff9942099fc700 x1862022294265984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768037.042268:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768037.042270:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768037.042274:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023cdc0. 00000400:00000200:0.0:1775768037.042279:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6d48 00000400:00000010:0.0:1775768037.042280:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6d48. 00000100:00000001:0.0:1775768037.042284:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768037.042287:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fc700 x1862022294265984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768037.042297:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fc700 x1862022294265984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768037.042304:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768037.042306:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768037.042308:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768037.042310:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768037.042315:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fc700 x1862022294265984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768037.042322:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768037.042326:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fc700 x1862022294265984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768058 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768037.042334:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768037.042337:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768037.042338:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768037.042340:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768037.042342:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768037.042347:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768037.042352:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768037.042357:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768037.042361:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768037.042366:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fc700 x1862022294265984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768058 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768037.042374:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294265984:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768037.042380:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768037.042383:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768033/real 1775768037] req@ffff9942099ff100 x1862022294266112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768037.042390:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768037.042392:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768037.042394:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023cf78. 00000400:00000200:0.0:1775768037.042396:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6aa0 00000400:00000010:0.0:1775768037.042398:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6aa0. 00000100:00000001:0.0:1775768037.042400:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768037.042403:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ff100 x1862022294266112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768037.042410:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ff100 x1862022294266112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768037.042417:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768037.042418:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768037.042420:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768037.042421:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768037.042425:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ff100 x1862022294266112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768037.042432:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768037.042435:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ff100 x1862022294266112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768058 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768037.042442:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768037.042445:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768037.042448:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768037.042450:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768037.042452:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768037.042457:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768037.042460:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768037.042464:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768037.042466:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768037.042470:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ff100 x1862022294266112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768058 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768037.042478:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294266112:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768037.042483:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768037.042486:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768033/real 1775768037] req@ffff9942099fd500 x1862022294265856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768037.042493:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768037.042494:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768037.042496:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c6e0. 00000400:00000200:0.0:1775768037.042499:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf63b8 00000400:00000010:0.0:1775768037.042501:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf63b8. 00000100:00000001:0.0:1775768037.042503:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768037.042505:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fd500 x1862022294265856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768037.042513:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fd500 x1862022294265856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768037.042519:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768037.042521:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768037.042523:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768037.042524:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768037.042528:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fd500 x1862022294265856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768037.042554:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768037.042559:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fd500 x1862022294265856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768058 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768037.042566:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768037.042568:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768037.042569:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768037.042571:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768037.042572:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768037.042575:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768037.042577:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768037.042581:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768037.042583:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768037.042587:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fd500 x1862022294265856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768058 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768037.042594:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294265856:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768037.042599:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768037.042603:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768033/real 1775768037] req@ffff9942099ff800 x1862022294265728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768037.042610:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768037.042611:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768037.042613:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c318. 00000400:00000200:0.0:1775768037.043645:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6440 00000400:00000010:0.0:1775768037.043647:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6440. 00000100:00000001:0.0:1775768037.043654:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768037.043663:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ff800 x1862022294265728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768037.043680:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ff800 x1862022294265728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768037.043689:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768037.043691:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768037.043693:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768037.043695:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768037.043699:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ff800 x1862022294265728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768037.043707:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768037.043712:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ff800 x1862022294265728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768058 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768037.043719:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768037.043722:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768037.043724:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768037.043725:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768037.043727:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768037.043730:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768037.043733:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768037.043737:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768037.043740:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768037.043744:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ff800 x1862022294265728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768058 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768037.043753:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294265728:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768037.043759:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768037.043763:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768033/real 1775768037] req@ffff9942099fc000 x1862022294265600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768037.043779:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768037.043780:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768037.043783:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023cc08. 00000400:00000200:0.0:1775768037.043786:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf67f8 00000400:00000010:0.0:1775768037.043788:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf67f8. 00000100:00000001:0.0:1775768037.043791:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768037.043795:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fc000 x1862022294265600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768037.043804:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fc000 x1862022294265600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768037.043811:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768037.043813:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768037.043815:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768037.043817:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768037.043821:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fc000 x1862022294265600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768037.043828:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768037.043833:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fc000 x1862022294265600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768058 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768037.043840:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768037.043843:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768037.043844:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768037.043846:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768037.043847:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768037.043849:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768037.043851:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768037.043855:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768037.043857:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768037.043861:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fc000 x1862022294265600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768058 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768037.043871:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294265600:192.168.202.156@tcp:250:kworker.0 00000100:00000001:0.0:1775768037.043876:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768037.043879:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768037.043883:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fc700 x1862022294265984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768058 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768037.043890:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768037.043893:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768037.043897:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543d800. 02000000:00000001:0.0:1775768037.043901:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768037.043904:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768037.043906:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:0.0:1775768037.043908:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768037.043911:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002bc00. 02000000:00000001:0.0:1775768037.043915:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768037.043917:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768037.043919:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fc700. 00000100:00000001:0.0:1775768037.043923:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768037.043925:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768037.043927:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768037.043931:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ff100 x1862022294266112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768058 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768037.043938:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768037.043940:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768037.043942:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543c400. 02000000:00000001:0.0:1775768037.043945:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768037.043946:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768037.043947:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:0.0:1775768037.043950:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768037.043952:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002b800. 02000000:00000001:0.0:1775768037.043954:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768037.043955:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768037.043956:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ff100. 00000100:00000001:0.0:1775768037.043960:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768037.043961:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768037.043963:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768037.043967:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fd500 x1862022294265856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768058 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768037.043974:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768037.043975:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768037.043976:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543e400. 02000000:00000001:0.0:1775768037.043979:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768037.043980:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768037.043981:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:0.0:1775768037.043984:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768037.043985:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002a800. 02000000:00000001:0.0:1775768037.043988:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768037.043989:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768037.043990:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fd500. 00000100:00000001:0.0:1775768037.044005:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768037.044006:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768037.044008:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768037.044011:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ff800 x1862022294265728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768058 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768037.044018:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768037.044019:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768037.044021:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543d400. 02000000:00000001:0.0:1775768037.044023:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768037.044024:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768037.044025:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:0.0:1775768037.044027:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768037.044029:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028000. 02000000:00000001:0.0:1775768037.044031:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768037.044032:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768037.044034:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ff800. 00000100:00000001:0.0:1775768037.044035:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768037.044036:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768037.044038:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768037.044040:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fc000 x1862022294265600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768058 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768037.044047:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768037.044048:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768037.044050:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543d000. 02000000:00000001:0.0:1775768037.044052:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768037.044053:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768037.044054:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:0.0:1775768037.044056:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768037.044057:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028400. 02000000:00000001:0.0:1775768037.044058:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768037.044059:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768037.044060:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fc000. 00000100:00000001:0.0:1775768037.044062:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768037.044063:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768037.044065:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768037.044073:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768037.044075:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768037.044077:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775768037.044079:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768037.105241:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768037.105247:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768037.105253:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768037.105255:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768037.105259:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768037.105260:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768037.105263:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768037.105265:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768037.105268:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768037.105269:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768037.105290:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768037.105293:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768037.105296:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768037.169260:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768037.169268:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768037.169270:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768037.169277:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768037.169281:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768037.169285:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768037.169288:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768037.169291:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768037.169294:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768038.065279:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768038.065279:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768038.065288:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768038.065288:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768038.065291:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768038.065291:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768038.065294:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768038.065299:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:0.0:1775768038.065300:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775768038.065300:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768038.065301:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768038.065307:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768038.065308:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768038.065310:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768038.065313:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768038.065315:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768038.065315:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768038.065316:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768038.065322:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768038.065325:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768038.065327:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768038.065330:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768038.065333:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768038.065337:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768038.065337:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768038.065339:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768038.065341:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768038.129296:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768038.129302:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768038.129307:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768038.129309:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768038.129312:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768038.129313:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768038.129316:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768038.129318:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768038.129321:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768038.129321:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768038.129341:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768038.129343:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768038.129346:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768038.193287:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768038.193295:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768038.193297:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768038.193305:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768038.193309:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768038.193314:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768038.193317:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768038.193320:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768038.193324:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768039.089241:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768039.089247:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768039.089249:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768039.089254:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775768039.089255:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768039.089257:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768039.089259:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768039.089262:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768039.089263:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768039.089264:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768039.089265:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768039.089268:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768039.089270:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:0.0:1775768039.089273:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768039.089274:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768039.089275:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768039.089278:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768039.089278:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768039.089280:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768039.089283:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768039.089284:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768039.089284:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768039.089288:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768039.089293:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768039.089295:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768039.089298:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768039.089302:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768039.089309:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768039.089315:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775768039.089318:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775768039.089321:0:8:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775768039.089325:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775768039.089332:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5291 00000400:00000200:2.0:1775768039.089343:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768039.089357:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775768039.089360:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775768039.089364:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768039.089367:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768039.089369:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768039.089371:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775768039.089374:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775768039.089376:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:2.0:1775768039.089378:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775768039.089381:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:2.0:1775768039.089384:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:2.0:1775768039.089386:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:2.0:1775768039.089390:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768039.089392:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768039.089394:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768039.089396:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775768039.089399:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:2.0:1775768039.089403:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775768039.089405:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768039.089408:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775768039.089412:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768039.089415:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768039.089417:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775768039.089420:0:8:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775768039.089426:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dff100. 00000020:00000040:2.0:1775768039.089430:0:8:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:2.0:1775768039.089435:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775768039.089437:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768039.089440:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775768039.089444:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775768039.089449:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775768039.089459:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1f000. 00000100:00000001:2.0:1775768039.089466:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768039.089487:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dff100 x1862022294266240/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775768039.089499:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775768039.089504:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768039.089511:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3e64ae1] to pc [ptlrpcd_rcv+-1] req@ffff994207dff100 x1862022294266240/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768039.089552:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768039.089561:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768039.089565:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775768039.089567:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775768039.089568:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775768039.089571:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775768039.089574:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5291 00000400:00000200:2.0:1775768039.089578:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768039.089582:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775768039.089585:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775768039.089589:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768039.089592:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768039.089594:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768039.089596:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775768039.089599:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775768039.089616:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:2.0:1775768039.089618:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775768039.089621:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:2.0:1775768039.089623:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:2.0:1775768039.089625:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:2.0:1775768039.089629:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768039.089631:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768039.089633:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768039.089635:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775768039.089638:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00080000:2.0:1775768039.089646:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775768039.089649:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768039.089651:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775768039.089660:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768039.089674:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768039.089677:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775768039.089682:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775768039.089686:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768039.089688:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775768039.089694:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfdc00. 00000020:00000040:2.0:1775768039.089697:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:2.0:1775768039.089699:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775768039.089701:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768039.089703:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775768039.089704:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775768039.089706:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775768039.089711:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1fc00. 00000100:00000001:2.0:1775768039.089715:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768039.089734:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfdc00 x1862022294266368/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775768039.089744:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775768039.089746:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768039.089751:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003e25603c] to pc [ptlrpcd_rcv+-1] req@ffff994207dfdc00 x1862022294266368/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768039.089760:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768039.089763:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768039.089767:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775768039.089768:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775768039.089770:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775768039.089773:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775768039.089776:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5291 00000400:00000200:2.0:1775768039.089780:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768039.089783:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775768039.089785:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775768039.089787:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768039.089789:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768039.089792:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768039.089794:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775768039.089796:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775768039.089797:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:2.0:1775768039.089799:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775768039.089801:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:2.0:1775768039.089802:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:2.0:1775768039.089804:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:2.0:1775768039.089807:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768039.089810:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768039.089811:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768039.089813:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775768039.089815:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:2.0:1775768039.089817:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775768039.089820:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768039.089822:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775768039.089825:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768039.089827:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768039.089829:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775768039.089832:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775768039.089834:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768039.089836:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775768039.089841:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfc700. 00000020:00000040:2.0:1775768039.089843:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:2.0:1775768039.089846:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775768039.089848:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768039.089849:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775768039.089851:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1775768039.089852:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1775768039.089853:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768039.089858:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 02000000:00000010:2.0:1775768039.089860:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1ec00. 00000100:00000001:0.0:1775768039.089863:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1775768039.089865:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768039.089873:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dff100 x1862022294266240/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768039.089873:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfc700 x1862022294266496/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775768039.089908:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768039.089909:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768039.089912:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768039.089913:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768039.089918:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a176dd1] to pc [ptlrpcd_rcv+-1] req@ffff994207dfc700 x1862022294266496/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1775768039.089920:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768039.089925:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768039.089932:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768039.089935:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768039.089940:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294266240:192.168.202.156@tcp:250:kworker.0 00000100:00080000:2.0:1775768039.089941:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775768039.089949:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00080000:2.0:1775768039.089950:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1775768039.089953:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775768039.089955:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:0.0:1775768039.089956:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1775768039.089960:0:8:0:(import.c:484:import_select_connection()) Process entered 02000000:00000001:0.0:1775768039.089961:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768039.089964:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5291 02000000:00000001:0.0:1775768039.089970:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:2.0:1775768039.089972:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768039.089977:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 02000000:00000010:0.0:1775768039.089978:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220028000. 00000100:00080000:2.0:1775768039.089980:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 02000000:00000001:0.0:1775768039.089983:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768039.089987:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768039.089990:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1775768039.089994:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768039.089999:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775768039.090009:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000040:2.0:1775768039.090009:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768039.090015:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775768039.090018:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775768039.090019:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:2.0:1775768039.090021:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000200:0.0:1775768039.090022:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294266240, portal 25 00000020:00000040:2.0:1775768039.090024:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000100:00000001:0.0:1775768039.090026:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000020:00000001:2.0:1775768039.090028:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:0.0:1775768039.090029:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611848960 : -112966097702656 : ffff994207dff100) 00000020:00000001:2.0:1775768039.090035:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:0.0:1775768039.090040:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dff100 x1862022294266240/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768064 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775768039.090044:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768039.090049:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768039.090055:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768039.090060:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1775768039.090060:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775768039.090065:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00000040:0.0:1775768039.090070:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000100:00080000:2.0:1775768039.090072:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000400:00000010:0.0:1775768039.090074:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000001:2.0:1775768039.090078:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1775768039.090080:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294266240, offset 0 02000000:00000001:2.0:1775768039.090083:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775768039.090088:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768039.090093:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768039.090094:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00000001:2.0:1775768039.090100:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775768039.090103:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:2.0:1775768039.090107:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768039.090109:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768039.090110:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775768039.090116:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfe300. 00000400:00000200:0.0:1775768039.090118:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000020:00000040:2.0:1775768039.090122:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00000400:00000200:0.0:1775768039.090126:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:2.0:1775768039.090126:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775768039.090129:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768039.090132:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:0.0:1775768039.090133:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 02000000:00000001:2.0:1775768039.090136:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775768039.090139:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768039.090140:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768039.090146:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 02000000:00000010:2.0:1775768039.090148:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1c800. 00000100:00000001:2.0:1775768039.090183:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768039.090192:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfe300 x1862022294266624/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1775768039.090198:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225253:225253:256:4294967295] 192.168.202.156@tcp LPNI seq info [225249:225249:8:4294967295] 00000100:00000001:2.0:1775768039.090201:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775768039.090206:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768039.090210:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000040:2.0:1775768039.090214:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c661fef] to pc [ptlrpcd_rcv+-1] req@ffff994207dfe300 x1862022294266624/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775768039.090230:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000100:00000001:2.0:1775768039.090230:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768039.090237:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768039.090243:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000200:0.0:1775768039.090244:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00000001:2.0:1775768039.090247:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775768039.090250:0:8:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000010:0.0:1775768039.090253:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000100:00000001:2.0:1775768039.090253:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775768039.090259:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5291 00000800:00000200:0.0:1775768039.090266:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000400:00000200:2.0:1775768039.090266:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000800:00000200:0.0:1775768039.090274:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000400:00000200:2.0:1775768039.090275:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775768039.090280:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000800:00000200:0.1:1775768039.090286:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000040:2.0:1775768039.090287:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000800:00000200:0.1:1775768039.090291:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1775768039.090298:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768039.090303:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000100:00000001:2.0:1775768039.090304:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768039.090309:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768039.090312:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775768039.090314:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775768039.090316:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:2.0:1775768039.090317:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775768039.090319:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:2.0:1775768039.090321:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:2.0:1775768039.090323:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:2.0:1775768039.090326:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768039.090329:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768039.090330:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768039.090332:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775768039.090335:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:2.0:1775768039.090338:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775768039.090341:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768039.090343:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775768039.090346:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768039.090349:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768039.090351:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775768039.090354:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:2.0:1775768039.090356:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768039.090358:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775768039.090362:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfed80. 00000020:00000040:2.0:1775768039.090364:0:8:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:2.0:1775768039.090367:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775768039.090369:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768039.090370:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775768039.090372:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775768039.090374:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775768039.090377:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1d400. 00000100:00000001:2.0:1775768039.090381:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768039.090386:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfed80 x1862022294266752/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775768039.090394:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775768039.090396:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768039.090400:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb9b9eae] to pc [ptlrpcd_rcv+-1] req@ffff994207dfed80 x1862022294266752/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768039.090409:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768039.090414:0:8:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5301) 00000400:00000200:0.0:1775768039.090986:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768039.090994:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768039.091001:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775768039.091004:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775768039.091009:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768039.091017:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dff100 x1862022294266240/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768064 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768039.091027:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768039.091031:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dff100 x1862022294266240/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768064 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768039.091037:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768039.091039:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768039.091047:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768039.091051:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768039.091055:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299965246 00000100:00000010:0.0:1775768039.091062:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942078a3d80. 00000100:00000001:0.0:1775768039.091069:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775768039.091081:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299965246 00000100:00000010:0.0:1775768039.091084:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942078a3c60. 00000100:00000001:0.0:1775768039.091088:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775768039.091091:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768039.091094:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768039.091096:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768039.091100:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768039/real 1775768039] req@ffff994207dff100 x1862022294266240/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768039.091109:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768039.091111:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768039.091113:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775768039.091115:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768039.091117:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768039.091119:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768039.091123:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dff100 x1862022294266240/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768039.091131:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dff100 x1862022294266240/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768039.091138:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768039.091139:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768039.091142:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768039.091143:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768039.091146:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dff100 x1862022294266240/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768039.091152:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768039.091183:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dff100 x1862022294266240/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768064 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768039.091196:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768039.091200:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768039.091201:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768039.091203:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768039.091205:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768039.091207:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768039.091209:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768039.091215:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768039.091217:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768039.091221:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dff100 x1862022294266240/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768064 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768039.091229:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294266240:192.168.202.156@tcp:250:kworker.0 00000100:00000001:0.0:1775768039.091235:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768039.091239:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfdc00 x1862022294266368/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768039.091247:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768039.091248:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768039.091251:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768039.091254:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768039.091256:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768039.091260:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294266368:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768039.091264:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768039.091267:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768039.091269:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768039.091272:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768039.091278:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002a800. 02000000:00000001:0.0:1775768039.091280:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768039.091286:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768039.091290:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775768039.091294:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768039.091299:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294266368, portal 10 00000100:00000001:0.0:1775768039.091301:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768039.091303:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611843584 : -112966097708032 : ffff994207dfdc00) 00000100:00000040:0.0:1775768039.091324:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfdc00 x1862022294266368/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768064 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768039.091330:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768039.091333:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768039.091336:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775768039.091339:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294266368, offset 0 00000400:00000200:0.0:1775768039.091344:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768039.091349:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768039.091354:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768039.091358:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768039.091362:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768039.091365:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768039.091374:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768039.091378:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225254:225254:256:4294967295] 192.168.202.156@tcp LPNI seq info [225250:225250:8:4294967295] 00000400:00000200:0.0:1775768039.091382:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768039.091390:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768039.091396:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768039.091399:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775768039.091404:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768039.091408:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775768039.091413:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768039.091416:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1775768039.091420:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768039.091422:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775768039.091425:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768039.091429:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768039.091432:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775768039.091434:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775768039.091437:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768039.091441:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfdc00 x1862022294266368/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768064 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768039.091447:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768039.091450:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfdc00 x1862022294266368/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768064 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768039.091457:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768039.091459:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768039.091463:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768039.091467:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:0.0:1775768039.091470:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768039.091472:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299963924 00000100:00000001:0.0:1775768039.091475:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299965246 00000100:00000001:0.0:1775768039.091477:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299963924 00000100:00000001:0.0:1775768039.091479:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299965246 00000100:00000001:0.0:1775768039.091481:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768039.091483:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768039.091485:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768039.091488:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768039/real 1775768039] req@ffff994207dfdc00 x1862022294266368/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768039.091495:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768039.091496:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768039.091498:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775768039.091501:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768039.091503:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768039.091505:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768039.091508:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfdc00 x1862022294266368/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768039.091536:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfdc00 x1862022294266368/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768039.091549:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768039.091551:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768039.091553:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768039.091554:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768039.091559:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfdc00 x1862022294266368/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768039.091566:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768039.091569:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfdc00 x1862022294266368/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768064 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768039.091575:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768039.091577:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768039.091579:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768039.091580:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768039.091581:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768039.091585:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768039.091587:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768039.091590:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768039.091592:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768039.091595:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfdc00 x1862022294266368/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768064 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768039.091602:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294266368:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768039.091606:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768039.091609:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768039.091612:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dff100 x1862022294266240/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768064 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768039.091618:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768039.091622:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768039.091627:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220028000. 02000000:00000001:0.0:1775768039.091632:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768039.091635:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768039.091637:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:0.0:1775768039.091640:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768039.091644:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1f000. 02000000:00000001:0.0:1775768039.091649:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768039.091653:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768039.091657:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dff100. 00000100:00000001:0.0:1775768039.091661:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768039.091663:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768039.091664:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768039.091667:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfdc00 x1862022294266368/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768064 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768039.091674:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768039.091675:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768039.091677:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002a800. 02000000:00000001:0.0:1775768039.091682:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768039.091685:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768039.091686:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:0.0:1775768039.091687:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768039.091688:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1fc00. 02000000:00000001:0.0:1775768039.091690:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768039.091691:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768039.091692:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfdc00. 00000100:00000001:0.0:1775768039.091694:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768039.091695:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768039.091698:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768039.091710:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768039.091712:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768039.091715:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775768039.091717:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768039.091718:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768039.091722:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfc700 x1862022294266496/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768039.091729:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768039.091732:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768039.091735:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768039.091736:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768039.091739:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768039.091742:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294266496:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768039.091746:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768039.091749:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768039.091750:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768039.091755:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768039.091759:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002a800. 02000000:00000001:0.0:1775768039.091761:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768039.091765:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768039.091770:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775768039.091773:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768039.091776:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294266496, portal 10 00000100:00000001:0.0:1775768039.091778:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768039.091779:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611838208 : -112966097713408 : ffff994207dfc700) 00000100:00000040:0.0:1775768039.091786:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfc700 x1862022294266496/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768064 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768039.091793:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768039.091795:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768039.091798:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775768039.091800:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294266496, offset 0 00000400:00000200:0.0:1775768039.091805:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768039.091808:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768039.091811:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768039.091814:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768039.091817:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768039.091820:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768039.091823:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768039.091826:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225255:225255:256:4294967295] 192.168.202.156@tcp LPNI seq info [225251:225251:8:4294967295] 00000400:00000200:0.0:1775768039.091830:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768039.091836:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768039.091841:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768039.091844:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775768039.091848:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768039.091851:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775768039.091855:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768039.091857:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1775768039.091862:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768039.091863:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775768039.091867:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768039.091871:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768039.091873:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775768039.091876:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775768039.091878:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768039.091882:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfc700 x1862022294266496/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768064 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768039.091889:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768039.091891:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfc700 x1862022294266496/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768064 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768039.091897:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768039.091899:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768039.091901:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768039.091905:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:0.0:1775768039.091908:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768039.091910:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299963916 00000100:00000001:0.0:1775768039.091913:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299965246 00000100:00000001:0.0:1775768039.091915:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299963916 00000100:00000001:0.0:1775768039.091917:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299965246 00000100:00000001:0.0:1775768039.091919:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768039.091921:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768039.091923:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768039.091926:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768039/real 1775768039] req@ffff994207dfc700 x1862022294266496/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768039.091935:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768039.091937:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768039.091938:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775768039.091942:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768039.091943:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768039.091945:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768039.091947:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfc700 x1862022294266496/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768039.091955:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfc700 x1862022294266496/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768039.091960:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768039.091961:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768039.091963:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768039.091964:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768039.091967:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfc700 x1862022294266496/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768039.091973:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768039.091976:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfc700 x1862022294266496/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768064 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768039.091981:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768039.091983:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768039.091984:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768039.091986:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768039.091987:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768039.091992:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768039.091994:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768039.091998:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768039.092001:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768039.092005:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfc700 x1862022294266496/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768064 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768039.092012:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294266496:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768039.092016:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768039.092019:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfe300 x1862022294266624/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768039.092024:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768039.092026:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768039.092029:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768039.092030:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768039.092033:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768039.092035:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294266624:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768039.092038:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768039.092039:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768039.092041:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768039.092042:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768039.092045:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220028000. 02000000:00000001:0.0:1775768039.092047:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768039.092050:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768039.092054:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775768039.092057:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768039.092062:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294266624, portal 4 00000100:00000001:0.0:1775768039.092064:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768039.092065:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611845376 : -112966097706240 : ffff994207dfe300) 00000100:00000040:0.0:1775768039.092070:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfe300 x1862022294266624/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768064 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768039.092076:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768039.092079:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768039.092082:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775768039.092084:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294266624, offset 0 00000400:00000200:0.0:1775768039.092089:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768039.092092:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768039.092095:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768039.092098:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768039.092101:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768039.092103:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768039.092106:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768039.092110:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225256:225256:256:4294967295] 192.168.202.156@tcp LPNI seq info [225252:225252:8:4294967295] 00000400:00000200:0.0:1775768039.092114:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768039.092120:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768039.092124:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768039.092127:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775768039.092132:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768039.092134:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775768039.092138:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768039.092140:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1775768039.092143:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768039.092145:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775768039.092148:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768039.092151:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768039.092153:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775768039.092185:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775768039.092187:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768039.092191:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfe300 x1862022294266624/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768064 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768039.092199:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768039.092201:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfe300 x1862022294266624/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768064 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768039.092207:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768039.092209:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768039.092226:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768039.092230:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:0.0:1775768039.092233:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768039.092236:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299963940 00000100:00000001:0.0:1775768039.092239:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299965246 00000100:00000001:0.0:1775768039.092241:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299963940 00000100:00000001:0.0:1775768039.092243:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299965246 00000100:00000001:0.0:1775768039.092245:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768039.092247:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768039.092249:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768039.092252:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768039/real 1775768039] req@ffff994207dfe300 x1862022294266624/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768039.092259:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768039.092261:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768039.092262:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775768039.092265:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768039.092267:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768039.092269:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768039.092272:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfe300 x1862022294266624/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768039.092279:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfe300 x1862022294266624/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768039.092284:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768039.092285:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768039.092287:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768039.092288:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768039.092292:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfe300 x1862022294266624/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768039.092297:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768039.092300:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfe300 x1862022294266624/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768064 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768039.092306:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768039.092309:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768039.092311:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768039.092312:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768039.092314:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768039.092318:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768039.092321:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768039.092324:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768039.092327:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768039.092330:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfe300 x1862022294266624/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768064 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768039.092336:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294266624:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768039.092340:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768039.092345:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfed80 x1862022294266752/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768039.092351:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768039.092353:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768039.092355:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768039.092357:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768039.092359:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768039.092362:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294266752:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768039.092365:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768039.092367:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768039.092369:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768039.092371:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768039.092374:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002b800. 02000000:00000001:0.0:1775768039.092376:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768039.092379:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768039.092382:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775768039.092385:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768039.092387:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294266752, portal 4 00000100:00000001:0.0:1775768039.092391:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768039.092393:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611848064 : -112966097703552 : ffff994207dfed80) 00000100:00000040:0.0:1775768039.092397:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfed80 x1862022294266752/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768064 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768039.092403:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768039.092405:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768039.092407:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775768039.092410:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294266752, offset 0 00000400:00000200:0.0:1775768039.092414:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768039.092417:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768039.092420:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768039.092423:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768039.092426:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768039.092428:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768039.092430:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768039.092434:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225257:225257:256:4294967295] 192.168.202.156@tcp LPNI seq info [225253:225253:8:4294967295] 00000400:00000200:0.0:1775768039.092439:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768039.092444:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768039.092448:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768039.092451:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775768039.092455:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768039.092458:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775768039.092462:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768039.092464:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1775768039.092468:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768039.092471:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775768039.092475:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768039.092478:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768039.092481:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775768039.092483:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775768039.092485:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768039.092488:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfed80 x1862022294266752/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768064 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768039.092502:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768039.092505:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfed80 x1862022294266752/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768064 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768039.092511:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768039.092512:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768039.092527:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768039.092530:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:0.0:1775768039.092534:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768039.092536:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299963943 00000100:00000001:0.0:1775768039.092539:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299965246 00000100:00000001:0.0:1775768039.092541:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299963943 00000100:00000001:0.0:1775768039.092543:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299965246 00000100:00000001:0.0:1775768039.092545:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768039.092547:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768039.092548:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768039.092552:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768039/real 1775768039] req@ffff994207dfed80 x1862022294266752/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768039.092559:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768039.092561:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768039.092563:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775768039.092565:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768039.092567:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768039.092568:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768039.092571:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfed80 x1862022294266752/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768039.092579:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfed80 x1862022294266752/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768039.092585:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768039.092587:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768039.092589:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768039.092590:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768039.092594:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfed80 x1862022294266752/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768039.092600:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768039.092603:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfed80 x1862022294266752/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768064 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768039.092609:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768039.092613:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768039.092615:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768039.092616:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768039.092618:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768039.092625:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768039.092626:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768039.092630:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768039.092633:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768039.092637:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfed80 x1862022294266752/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768064 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768039.092645:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294266752:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768039.092648:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768039.092650:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768039.092652:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfc700 x1862022294266496/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768064 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768039.092659:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768039.092660:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768039.092662:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002a800. 02000000:00000001:0.0:1775768039.092666:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768039.092668:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768039.092670:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:0.0:1775768039.092673:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768039.092674:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1ec00. 02000000:00000001:0.0:1775768039.092679:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768039.092681:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768039.092683:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfc700. 00000100:00000001:0.0:1775768039.092687:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768039.092688:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768039.092690:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768039.092694:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfe300 x1862022294266624/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768064 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768039.092701:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768039.092702:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768039.092704:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220028000. 02000000:00000001:0.0:1775768039.092708:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768039.092709:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768039.092710:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:0.0:1775768039.092712:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768039.092714:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1c800. 02000000:00000001:0.0:1775768039.092717:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768039.092719:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768039.092720:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfe300. 00000100:00000001:0.0:1775768039.092723:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768039.092724:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768039.092725:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768039.092727:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfed80 x1862022294266752/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768064 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768039.092733:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768039.092736:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768039.092737:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002b800. 02000000:00000001:0.0:1775768039.092740:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768039.092741:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768039.092741:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:0.0:1775768039.092743:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768039.092745:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1d400. 02000000:00000001:0.0:1775768039.092750:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768039.092752:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768039.092754:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfed80. 00000100:00000001:0.0:1775768039.092757:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768039.092758:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768039.092761:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768039.092767:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768039.092769:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768039.092772:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775768039.092773:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768039.153244:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768039.153249:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768039.153254:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768039.153256:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768039.153259:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768039.153261:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768039.153264:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768039.153265:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768039.153268:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768039.153270:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768039.153290:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768039.153293:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768039.153296:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768039.217245:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768039.217253:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768039.217255:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768039.217262:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768039.217266:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768039.217270:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768039.217273:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768039.217275:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768039.217278:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768040.113259:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768040.113260:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768040.113267:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768040.113267:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768040.113271:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768040.113271:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768040.113272:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768040.113277:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:0.0:1775768040.113278:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775768040.113278:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768040.113279:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768040.113284:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768040.113285:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1775768040.113285:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775768040.113288:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768040.113290:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768040.113290:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768040.113292:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768040.113293:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768040.113296:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768040.113300:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768040.113305:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768040.113309:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768040.113312:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768040.113312:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768040.113316:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768040.113319:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768040.177265:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768040.177271:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768040.177277:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768040.177278:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768040.177281:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768040.177282:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768040.177284:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768040.177285:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768040.177288:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768040.177289:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768040.177304:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768040.177306:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768040.177309:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768040.241239:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768040.241245:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768040.241246:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768040.241251:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768040.241253:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768040.241256:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768040.241258:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768040.241260:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768040.241262:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768040.433263:0:14953:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4299965246, at 4299965248 00000100:00000040:1.0:1775768040.433263:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299965246, at 4299965248 00000100:00000010:0.0:1775768040.433278:0:14953:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3d80. 00000100:00000010:1.0:1775768040.433284:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3c60. 00000100:00000001:3.0:1775768041.137223:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768041.137226:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768041.137229:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768041.137230:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768041.137231:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768041.137232:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768041.137234:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:0.0:1775768041.137237:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775768041.137237:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768041.137238:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768041.137241:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768041.137243:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768041.137243:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1775768041.137245:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768041.137248:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768041.137248:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768041.137248:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768041.137252:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768041.137253:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768041.137257:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768041.137260:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768041.137260:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768041.137262:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768041.137264:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768041.137264:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768041.137267:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768041.137270:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768041.201253:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768041.201258:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768041.201262:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768041.201264:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768041.201267:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768041.201268:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768041.201271:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768041.201272:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768041.201274:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768041.201275:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768041.201297:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768041.201299:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768041.201301:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768041.265257:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768041.265263:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768041.265264:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768041.265268:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768041.265271:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768041.265275:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768041.265277:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768041.265279:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768041.265280:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768042.161258:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768042.161264:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768042.161266:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768042.161271:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768042.161274:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768042.161278:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768042.161280:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768042.161282:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768042.161282:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768042.161286:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768042.161287:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768042.161289:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768042.161293:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768042.161296:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768042.161298:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768042.161300:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768042.161301:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768042.161303:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768042.161317:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768042.161321:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768042.161323:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768042.161325:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775768042.161328:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768042.161331:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768042.161332:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768042.161333:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768042.161335:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768042.225268:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768042.225275:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768042.225277:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768042.225280:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768042.225284:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768042.225286:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768042.225287:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768042.225289:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768042.225291:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768042.225292:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768042.225295:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768042.225296:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768042.225298:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768042.225299:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768042.225301:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768042.225302:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768042.225345:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768042.225347:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768042.225350:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768042.289236:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768042.289243:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768042.289245:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768042.289250:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768042.289254:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768042.289258:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768042.289261:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768042.289264:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768042.289267:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768043.185261:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768043.185266:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768043.185269:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768043.185270:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768043.185271:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:3.0:1775768043.185274:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768043.185275:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768043.185276:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768043.185279:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768043.185280:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768043.185280:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1775768043.185285:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:2.0:1775768043.185286:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768043.185286:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768043.185289:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768043.185292:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768043.185293:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768043.185293:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768043.185296:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768043.185298:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768043.185299:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768043.185303:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768043.185305:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768043.185308:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768043.185308:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768043.185311:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768043.185315:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768043.249267:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768043.249273:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768043.249279:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768043.249280:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768043.249282:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768043.249283:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768043.249285:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768043.249286:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768043.249288:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768043.249289:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768043.249326:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768043.249329:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768043.249332:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768043.313236:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768043.313241:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768043.313243:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768043.313247:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768043.313250:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768043.313259:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768043.313261:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768043.313268:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768043.313270:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768044.209262:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768044.209262:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768044.209264:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768044.209269:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768044.209270:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768044.209270:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768044.209274:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768044.209274:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768044.209274:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768044.209282:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:2.0:1775768044.209282:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775768044.209282:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768044.209289:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768044.209289:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768044.209289:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1775768044.209292:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775768044.209295:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768044.209295:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768044.209295:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768044.209302:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1775768044.209303:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775768044.209305:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768044.209307:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1775768044.209309:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768044.209309:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1775768044.209312:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775768044.209312:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768044.209317:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768044.209319:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775768044.209319:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768044.209323:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768044.209328:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768044.209329:0:13471:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5296 00000400:00000200:0.0:1775768044.209340:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768044.209348:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768044.209353:0:13471:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768044.209358:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768044.209371:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768044.209375:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768044.209377:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768044.209381:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768044.209384:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:0.0:1775768044.209386:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768044.209389:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:0.0:1775768044.209392:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:0.0:1775768044.209395:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:0.0:1775768044.209398:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768044.209400:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768044.209409:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768044.209411:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768044.209414:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:0.0:1775768044.209418:0:13471:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768044.209421:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768044.209424:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768044.209428:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768044.209431:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768044.209433:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775768044.209436:0:13471:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768044.209445:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fc000. 00000020:00000040:0.0:1775768044.209449:0:13471:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:0.0:1775768044.209453:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768044.209455:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768044.209458:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768044.209461:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768044.209478:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768044.209485:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220028000. 00000100:00000001:0.0:1775768044.209492:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768044.209502:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fc000 x1862022294266880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768044.209512:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768044.209518:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768044.209524:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008bcb9792] to pc [ptlrpcd_rcv+-1] req@ffff9942099fc000 x1862022294266880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768044.209549:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768044.209551:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768044.209554:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775768044.209555:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768044.209557:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768044.209559:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768044.209561:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5296 00000400:00000200:0.0:1775768044.209565:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768044.209569:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768044.209572:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1775768044.209576:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1775768044.209577:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768044.209581:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775768044.209581:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1775768044.209586:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768044.209586:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1775768044.209590:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768044.209595:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000040:1.0:1775768044.209596:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fc000 x1862022294266880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1775768044.209597:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:0.0:1775768044.209601:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768044.209605:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:0.0:1775768044.209611:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:0.0:1775768044.209616:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000001:1.0:1775768044.209618:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768044.209622:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768044.209624:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 02000000:00000001:1.0:1775768044.209627:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1775768044.209629:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1775768044.209631:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1775768044.209634:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000001:1.0:1775768044.209637:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768044.209638:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768044.209644:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00100000:1.0:1775768044.209648:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294266880:192.168.202.156@tcp:250:kworker.0 00000100:00080000:0.0:1775768044.209650:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768044.209654:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768044.209655:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768044.209657:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775768044.209659:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768044.209663:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768044.209663:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768044.209665:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768044.209669:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543f000. 02000000:00000001:1.0:1775768044.209671:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768044.209674:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768044.209676:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:1.0:1775768044.209679:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00100000:0.0:1775768044.209681:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000400:00000010:1.0:1775768044.209682:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000008:00000001:0.0:1775768044.209687:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768044.209689:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775768044.209690:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000010:0.0:1775768044.209693:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ff800. 00000020:00000040:0.0:1775768044.209696:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000100:00000200:1.0:1775768044.209697:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294266880, portal 25 02000000:00000001:0.0:1775768044.209698:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768044.209700:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768044.209701:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768044.209702:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768044.209704:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768044.209706:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768044.209708:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641196544 : -112966068355072 : ffff9942099fc000) 02000000:00000010:0.0:1775768044.209709:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002a800. 00000100:00000001:0.0:1775768044.209715:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768044.209720:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fc000 x1862022294266880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768069 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768044.209724:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ff800 x1862022294267008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768044.209751:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775768044.209752:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1775768044.209753:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768044.209758:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000aa4e8b71] to pc [ptlrpcd_rcv+-1] req@ffff9942099ff800 x1862022294267008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775768044.209759:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768044.209766:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000001:0.0:1775768044.209768:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768044.209771:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:1.0:1775768044.209772:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294266880, offset 0 00000100:00080000:0.0:1775768044.209775:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775768044.209778:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768044.209780:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768044.209783:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:1.0:1775768044.209784:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00080000:0.0:1775768044.209788:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5296 00000400:00000200:1.0:1775768044.209793:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768044.209798:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768044.209802:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768044.209806:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768044.209808:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768044.209811:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768044.209814:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225258:225258:256:4294967295] 192.168.202.156@tcp LPNI seq info [225254:225254:8:4294967295] 00000400:00000200:1.0:1775768044.209817:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768044.209826:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768044.209830:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768044.209835:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000800:00000010:1.0:1775768044.209836:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000400:00000200:0.0:1775768044.209842:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768044.209846:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000800:00000200:1.0:1775768044.209846:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768044.209850:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000100:00000040:0.0:1775768044.209855:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000800:00000200:1.1:1775768044.209860:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:0.0:1775768044.209861:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768044.209866:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000800:00000200:1.1:1775768044.209866:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000100:00000001:0.0:1775768044.209869:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768044.209874:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000800:00000100:1.0:1775768044.209874:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000020:00000040:0.0:1775768044.209875:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000800:00000010:1.0:1775768044.209877:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000020:00000001:0.0:1775768044.209878:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768044.209881:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:0.0:1775768044.209883:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000400:00000200:1.0:1775768044.209884:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000001:0.0:1775768044.209886:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:0.0:1775768044.209892:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000200:1.0:1775768044.209892:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000100:00000001:0.0:1775768044.209898:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1775768044.209899:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000100:00000040:0.0:1775768044.209903:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000010:1.0:1775768044.209903:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:0.0:1775768044.209907:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:1.0:1775768044.209907:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000020:00000040:0.0:1775768044.209912:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00000200:1.0:1775768044.209913:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fc000 x1862022294266880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768069 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768044.209919:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768044.209928:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768044.209929:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1775768044.209932:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:1.0:1775768044.209935:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fc000 x1862022294266880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768069 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1775768044.209938:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768044.209944:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768044.209946:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768044.209948:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775768044.209949:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000008:00100000:0.0:1775768044.209953:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1775768044.209955:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1775768044.209957:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768044.209960:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768044.209961:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768044.209965:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299970365 00000100:00000010:0.0:1775768044.209966:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fd500. 00000100:00000010:1.0:1775768044.209969:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba9c0. 00000020:00000040:0.0:1775768044.209970:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:0.0:1775768044.209975:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768044.209976:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768044.209977:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768044.209978:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1775768044.209979:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 02000000:00000001:0.0:1775768044.209980:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768044.209985:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299970365 02000000:00000010:0.0:1775768044.209986:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002bc00. 00000100:00000001:0.0:1775768044.209991:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775768044.209991:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba600. 00000100:00000001:1.0:1775768044.209997:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00100000:0.0:1775768044.210001:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fd500 x1862022294267136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775768044.210002:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768044.210008:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768044.210013:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1775768044.210017:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768044.210020:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768044.210020:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768044/real 1775768044] req@ffff9942099fc000 x1862022294266880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775768044.210029:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f246bb2b] to pc [ptlrpcd_rcv+-1] req@ffff9942099fd500 x1862022294267136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768044.210036:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768044.210039:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768044.210042:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000100:00000001:0.0:1775768044.210043:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768044.210045:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000100:00080000:0.0:1775768044.210046:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:1.0:1775768044.210049:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00080000:0.0:1775768044.210051:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775768044.210053:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775768044.210053:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00080000:0.0:1775768044.210056:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768044.210060:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000200:1.0:1775768044.210060:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fc000 x1862022294266880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768044.210066:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5296 00000400:00000200:0.0:1775768044.210073:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000200:1.0:1775768044.210079:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fc000 x1862022294266880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775768044.210080:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768044.210083:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768044.210089:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768044.210093:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768044.210097:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768044.210099:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768044.210100:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:1.0:1775768044.210101:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1775768044.210104:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:1.0:1775768044.210105:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000040:0.0:1775768044.210107:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000100:00000001:1.0:1775768044.210108:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1775768044.210110:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768044.210113:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000100:00100000:1.0:1775768044.210114:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fc000 x1862022294266880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775768044.210116:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:0.0:1775768044.210121:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:0.0:1775768044.210128:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768044.210131:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775768044.210131:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768044.210136:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000040:1.0:1775768044.210138:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fc000 x1862022294266880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768069 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768044.210140:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768044.210147:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00000001:1.0:1775768044.210152:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768044.210174:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768044.210176:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768044.210176:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768044.210178:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768044.210180:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768044.210182:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768044.210186:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768044.210189:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768044.210193:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fc000 x1862022294266880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768069 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768044.210199:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294266880:192.168.202.156@tcp:250:kworker.0 00000100:00000001:1.0:1775768044.210203:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1775768044.210205:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775768044.210206:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1775768044.210211:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768044.210211:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fc000 x1862022294266880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768069 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1775768044.210216:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768044.210220:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768044.210221:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768044.210225:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768044.210226:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:0.0:1775768044.210228:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000010:1.0:1775768044.210229:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543f000. 00000008:00100000:0.0:1775768044.210231:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 02000000:00000001:1.0:1775768044.210232:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000008:00000001:0.0:1775768044.210236:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768044.210236:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:0.0:1775768044.210238:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1775768044.210239:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:1.0:1775768044.210242:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000010:0.0:1775768044.210243:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ff100. 02000000:00000010:1.0:1775768044.210244:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028000. 00000020:00000040:0.0:1775768044.210247:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:1.0:1775768044.210249:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768044.210250:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768044.210252:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768044.210252:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768044.210255:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fc000. 00000100:00000001:0.0:1775768044.210256:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768044.210258:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768044.210261:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768044.210261:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768044.210264:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:0.0:1775768044.210266:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220029400. 00000100:00000001:1.0:1775768044.210266:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768044.210273:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768044.210275:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768044.210278:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768044.210282:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ff100 x1862022294267264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775768044.210285:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768044.210288:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768044.210290:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768044.210296:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ff800 x1862022294267008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768044.210300:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768044.210305:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768044.210307:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768044.210311:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768044.210313:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768044.210315:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1775768044.210316:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5e24158] to pc [ptlrpcd_rcv+-1] req@ffff9942099ff100 x1862022294267264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1775768044.210318:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768044.210324:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294267008:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768044.210332:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768044.210335:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768044.210338:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768044.210341:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1775768044.210343:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775768044.210343:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543f000. 02000000:00000001:1.0:1775768044.210348:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768044.210349:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1775768044.210353:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00080000:0.0:1775768044.210357:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000400:00000010:1.0:1775768044.210358:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000100:00000001:0.0:1775768044.210363:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000400:00000010:1.0:1775768044.210363:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00080000:0.0:1775768044.210366:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:1.0:1775768044.210368:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294267008, portal 10 00000100:00000001:0.0:1775768044.210372:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775768044.210373:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768044.210376:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641210880 : -112966068340736 : ffff9942099ff800) 00000100:00080000:0.0:1775768044.210378:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5296 00000400:00000200:0.0:1775768044.210385:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000040:1.0:1775768044.210388:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ff800 x1862022294267008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768069 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775768044.210393:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768044.210398:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768044.210406:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768044.210408:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1775768044.210410:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768044.210415:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000100:00000040:0.0:1775768044.210416:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000010:1.0:1775768044.210420:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000001:0.0:1775768044.210421:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768044.210426:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000200:1.0:1775768044.210427:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294267008, offset 0 00000020:00000040:0.0:1775768044.210428:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:0.0:1775768044.210432:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768044.210436:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000400:00000200:1.0:1775768044.210436:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000020:00000001:0.0:1775768044.210440:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000400:00000200:1.0:1775768044.210441:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:0.0:1775768044.210446:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000400:00000200:1.0:1775768044.210447:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768044.210453:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:0.0:1775768044.210454:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768044.210456:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768044.210459:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768044.210463:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000200:1.0:1775768044.210464:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768044.210482:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000020:00000040:0.0:1775768044.210483:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000400:00000200:1.0:1775768044.210488:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00080000:0.0:1775768044.210492:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000400:00000200:1.0:1775768044.210496:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225259:225259:256:4294967295] 192.168.202.156@tcp LPNI seq info [225255:225255:8:4294967295] 00000100:00000001:0.0:1775768044.210498:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768044.210503:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1775768044.210507:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:0.0:1775768044.210509:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768044.210513:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768044.210518:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:1.0:1775768044.210519:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000008:00100000:0.0:1775768044.210523:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:0.0:1775768044.210528:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775768044.210529:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00000001:0.0:1775768044.210534:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1775768044.210536:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000100:00000010:0.0:1775768044.210540:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fc700. 00000800:00000200:1.0:1775768044.210545:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000040:0.0:1775768044.210547:0:13471:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00000800:00000200:1.0:1775768044.210551:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 02000000:00000001:0.0:1775768044.210552:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768044.210556:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768044.210560:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000800:00000200:1.1:1775768044.210560:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 02000000:00000001:0.0:1775768044.210563:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:1.1:1775768044.210564:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 02000000:00000001:0.0:1775768044.210568:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1775768044.210571:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768044.210574:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 02000000:00000010:0.0:1775768044.210576:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220028000. 00000400:00000200:1.0:1775768044.210578:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:0.0:1775768044.210581:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768044.210587:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000100:00100000:0.0:1775768044.210592:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fc700 x1862022294267392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775768044.210592:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768044.210597:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768044.210603:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1775768044.210608:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000200:1.0:1775768044.210611:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ff800 x1862022294267008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768069 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768044.210613:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768044.210622:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003283b20a] to pc [ptlrpcd_rcv+-1] req@ffff9942099fc700 x1862022294267392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768044.210628:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768044.210634:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ff800 x1862022294267008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768069 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768044.210641:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768044.210647:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768044.210648:0:13471:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5306) 00000100:00000001:1.0:1775768044.210649:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768044.210654:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768044.210657:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:1.0:1775768044.210660:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768044.210662:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299969043 00000100:00000001:1.0:1775768044.210664:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299970365 00000100:00000001:1.0:1775768044.210666:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299969043 00000100:00000001:1.0:1775768044.210668:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299970365 00000100:00000001:1.0:1775768044.210670:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768044.210672:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768044.210673:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768044.210676:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768044/real 1775768044] req@ffff9942099ff800 x1862022294267008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768044.210682:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768044.210684:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768044.210686:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768044.210688:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768044.210689:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768044.210692:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768044.210695:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ff800 x1862022294267008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768044.210700:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ff800 x1862022294267008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768044.210704:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768044.210705:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768044.210706:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768044.210707:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768044.210709:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ff800 x1862022294267008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768044.210716:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768044.210720:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ff800 x1862022294267008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768069 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768044.210726:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768044.210729:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768044.210730:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768044.210732:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768044.210733:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768044.210736:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768044.210739:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768044.210743:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768044.210745:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768044.210750:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ff800 x1862022294267008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768069 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768044.210759:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294267008:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768044.210763:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768044.210766:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fd500 x1862022294267136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768044.210778:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768044.210779:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768044.210781:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768044.210782:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768044.210783:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768044.210785:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294267136:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768044.210787:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768044.210789:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768044.210790:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768044.210792:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768044.210794:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543c800. 02000000:00000001:1.0:1775768044.210796:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768044.210798:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768044.210800:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000400:00000010:1.0:1775768044.210803:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000200:1.0:1775768044.210805:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294267136, portal 10 00000100:00000001:1.0:1775768044.210806:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768044.210807:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641201920 : -112966068349696 : ffff9942099fd500) 00000100:00000040:1.0:1775768044.210811:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fd500 x1862022294267136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768069 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768044.210816:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768044.210817:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768044.210819:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768044.210820:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294267136, offset 0 00000400:00000200:1.0:1775768044.210823:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768044.210825:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768044.210828:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768044.210830:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768044.210831:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768044.210833:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768044.210835:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768044.210837:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225260:225260:256:4294967295] 192.168.202.156@tcp LPNI seq info [225256:225256:8:4294967295] 00000400:00000200:1.0:1775768044.210840:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768044.210844:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768044.210847:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768044.210850:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000800:00000200:1.0:1775768044.210852:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768044.210854:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775768044.210857:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768044.210859:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1775768044.210862:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768044.210863:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768044.210865:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768044.210867:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768044.210869:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768044.210870:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768044.210872:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768044.210874:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fd500 x1862022294267136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768069 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768044.210879:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768044.210881:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fd500 x1862022294267136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768069 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768044.210886:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768044.210887:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768044.210889:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768044.210891:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:1.0:1775768044.210892:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768044.210894:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299969035 00000100:00000001:1.0:1775768044.210895:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299970365 00000100:00000001:1.0:1775768044.210897:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299969035 00000100:00000001:1.0:1775768044.210898:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299970365 00000100:00000001:1.0:1775768044.210900:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768044.210902:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768044.210903:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768044.210905:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768044/real 1775768044] req@ffff9942099fd500 x1862022294267136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768044.210910:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768044.210911:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768044.210912:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768044.210914:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768044.210915:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768044.210916:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768044.210918:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fd500 x1862022294267136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768044.210923:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fd500 x1862022294267136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768044.210927:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768044.210928:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768044.210929:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768044.210930:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768044.210932:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fd500 x1862022294267136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768044.210936:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768044.210938:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fd500 x1862022294267136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768069 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768044.210942:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768044.210943:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768044.210944:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768044.210945:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768044.210946:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768044.210948:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768044.210949:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768044.210952:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768044.210953:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768044.210956:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fd500 x1862022294267136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768069 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768044.210960:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294267136:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768044.210963:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768044.210964:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768044.210965:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ff800 x1862022294267008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768069 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768044.210969:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768044.210970:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768044.210971:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543f000. 02000000:00000001:1.0:1775768044.210973:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768044.210974:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768044.210974:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:1.0:1775768044.210976:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768044.210977:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002a800. 02000000:00000001:1.0:1775768044.210979:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768044.210980:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768044.210981:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ff800. 00000100:00000001:1.0:1775768044.210983:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768044.210983:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768044.210985:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768044.210987:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fd500 x1862022294267136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768069 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768044.210991:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768044.210991:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768044.210992:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543c800. 02000000:00000001:1.0:1775768044.210993:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768044.210994:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768044.210995:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:1.0:1775768044.210996:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768044.210997:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002bc00. 02000000:00000001:1.0:1775768044.210998:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768044.210999:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768044.211000:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fd500. 00000100:00000001:1.0:1775768044.211002:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768044.211002:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768044.211003:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768044.211006:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768044.211007:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768044.211008:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768044.211009:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768044.211010:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768044.211012:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ff100 x1862022294267264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768044.211017:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768044.211018:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768044.211019:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768044.211020:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768044.211021:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768044.211023:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294267264:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768044.211025:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768044.211026:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768044.211027:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768044.211029:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768044.211030:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543c800. 02000000:00000001:1.0:1775768044.211032:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768044.211034:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768044.211036:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000400:00000010:1.0:1775768044.211038:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000200:1.0:1775768044.211040:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294267264, portal 4 00000100:00000001:1.0:1775768044.211041:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768044.211042:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641209088 : -112966068342528 : ffff9942099ff100) 00000100:00000040:1.0:1775768044.211045:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ff100 x1862022294267264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768069 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768044.211053:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768044.211055:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768044.211056:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768044.211058:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294267264, offset 0 00000400:00000200:1.0:1775768044.211060:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768044.211062:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768044.211064:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768044.211065:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768044.211067:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768044.211068:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768044.211070:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768044.211072:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225261:225261:256:4294967295] 192.168.202.156@tcp LPNI seq info [225257:225257:8:4294967295] 00000400:00000200:1.0:1775768044.211075:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768044.211079:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768044.211082:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768044.211083:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000800:00000200:1.0:1775768044.211086:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768044.211088:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775768044.211090:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768044.211103:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1775768044.211106:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768044.211107:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768044.211109:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768044.211112:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768044.211114:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768044.211115:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768044.211117:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768044.211119:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ff100 x1862022294267264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768069 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768044.211124:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768044.211126:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ff100 x1862022294267264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768069 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768044.211134:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768044.211135:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768044.211138:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768044.211140:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:1.0:1775768044.211142:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768044.211143:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299969058 00000100:00000001:1.0:1775768044.211145:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299970365 00000100:00000001:1.0:1775768044.211146:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299969058 00000100:00000001:1.0:1775768044.211148:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299970365 00000100:00000001:1.0:1775768044.211149:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768044.211151:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768044.211152:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768044.211170:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768044/real 1775768044] req@ffff9942099ff100 x1862022294267264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768044.211176:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768044.211176:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768044.211177:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768044.211179:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768044.211180:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768044.211182:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768044.211184:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ff100 x1862022294267264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768044.211189:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ff100 x1862022294267264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768044.211194:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768044.211194:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768044.211196:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768044.211196:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768044.211198:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ff100 x1862022294267264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768044.211203:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768044.211205:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ff100 x1862022294267264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768069 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768044.211210:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768044.211211:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768044.211212:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768044.211213:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768044.211214:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768044.211216:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768044.211218:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768044.211220:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768044.211221:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768044.211223:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ff100 x1862022294267264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768069 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768044.211228:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294267264:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768044.211230:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768044.211232:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fc700 x1862022294267392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768044.211237:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768044.211238:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768044.211239:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768044.211240:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768044.211241:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768044.211243:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294267392:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768044.211245:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768044.211246:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768044.211248:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768044.211249:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768044.211251:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543f000. 02000000:00000001:1.0:1775768044.211252:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768044.211254:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768044.211256:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000400:00000010:1.0:1775768044.211258:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000200:1.0:1775768044.211260:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294267392, portal 4 00000100:00000001:1.0:1775768044.211261:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768044.211262:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641198336 : -112966068353280 : ffff9942099fc700) 00000100:00000040:1.0:1775768044.211265:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fc700 x1862022294267392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768069 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768044.211270:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768044.211271:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768044.211272:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768044.211274:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294267392, offset 0 00000400:00000200:1.0:1775768044.211276:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768044.211278:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768044.211280:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768044.211282:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768044.211284:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768044.211285:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768044.211287:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768044.211289:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225262:225262:256:4294967295] 192.168.202.156@tcp LPNI seq info [225258:225258:8:4294967295] 00000400:00000200:1.0:1775768044.211292:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768044.211296:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768044.211299:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768044.211301:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000800:00000200:1.0:1775768044.211303:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768044.211305:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775768044.211308:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768044.211309:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1775768044.211312:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768044.211313:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768044.211315:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768044.211317:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768044.211319:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768044.211320:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768044.211322:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768044.211324:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fc700 x1862022294267392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768069 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768044.211329:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768044.211331:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fc700 x1862022294267392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768069 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768044.211335:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768044.211336:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768044.211337:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768044.211339:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:1.0:1775768044.211341:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768044.211342:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299969062 00000100:00000001:1.0:1775768044.211344:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299970365 00000100:00000001:1.0:1775768044.211345:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299969062 00000100:00000001:1.0:1775768044.211347:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299970365 00000100:00000001:1.0:1775768044.211348:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768044.211349:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768044.211351:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768044.211352:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768044/real 1775768044] req@ffff9942099fc700 x1862022294267392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768044.211357:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768044.211358:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768044.211359:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768044.211360:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768044.211361:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768044.211363:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768044.211365:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fc700 x1862022294267392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768044.211373:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fc700 x1862022294267392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768044.211379:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768044.211381:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768044.211383:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768044.211384:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768044.211387:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fc700 x1862022294267392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768044.211395:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768044.211398:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fc700 x1862022294267392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768069 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768044.211405:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768044.211408:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768044.211410:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768044.211411:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768044.211413:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768044.211416:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768044.211419:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768044.211422:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768044.211425:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768044.211428:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fc700 x1862022294267392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768069 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768044.211435:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294267392:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768044.211438:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768044.211439:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768044.211441:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ff100 x1862022294267264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768069 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768044.211446:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768044.211447:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768044.211448:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543c800. 02000000:00000001:1.0:1775768044.211450:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768044.211451:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768044.211452:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:1.0:1775768044.211453:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768044.211455:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029400. 02000000:00000001:1.0:1775768044.211456:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768044.211457:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768044.211458:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ff100. 00000100:00000001:1.0:1775768044.211460:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768044.211461:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768044.211462:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768044.211464:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fc700 x1862022294267392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768069 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768044.211483:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768044.211484:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768044.211485:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543f000. 02000000:00000001:1.0:1775768044.211487:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768044.211488:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768044.211490:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:1.0:1775768044.211492:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768044.211494:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028000. 02000000:00000001:1.0:1775768044.211496:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768044.211497:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768044.211498:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fc700. 00000100:00000001:1.0:1775768044.211500:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768044.211502:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768044.211504:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768044.211507:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768044.211509:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768044.211511:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768044.211512:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768044.273257:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768044.273262:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768044.273266:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768044.273268:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768044.273270:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768044.273271:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768044.273274:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768044.273275:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768044.273277:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768044.273278:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768044.273299:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768044.273302:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768044.273305:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768044.337221:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768044.337227:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768044.337228:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768044.337233:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768044.337236:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768044.337239:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768044.337241:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768044.337243:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768044.337246:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768045.233249:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768045.233250:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768045.233256:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768045.233257:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768045.233258:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768045.233259:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768045.233262:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:1.0:1775768045.233265:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:2.0:1775768045.233266:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768045.233266:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768045.233270:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768045.233272:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768045.233273:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1775768045.233274:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768045.233278:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768045.233279:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768045.233279:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768045.233283:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768045.233284:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768045.233287:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768045.233290:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768045.233291:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768045.233293:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768045.233296:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768045.233296:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768045.233299:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768045.233301:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768045.297275:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768045.297280:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768045.297284:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768045.297285:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768045.297288:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768045.297289:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768045.297291:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768045.297292:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768045.297296:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768045.297297:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768045.297334:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768045.297337:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768045.297341:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768045.361229:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768045.361241:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768045.361242:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768045.361246:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768045.361249:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768045.361253:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768045.361255:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768045.361257:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768045.361259:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768045.553246:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299970365, at 4299970368 00000100:00000040:0.0:1775768045.553250:0:14953:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4299970365, at 4299970368 00000100:00000010:1.0:1775768045.553259:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cba600. 00000100:00000010:0.0:1775768045.553262:0:14953:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cba9c0. 00000100:00000001:1.0:1775768046.257230:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768046.257232:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768046.257235:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768046.257237:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768046.257239:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768046.257239:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768046.257242:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768046.257242:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768046.257245:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768046.257245:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:2.0:1775768046.257247:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768046.257251:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1775768046.257252:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775768046.257252:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768046.257257:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768046.257257:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768046.257258:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768046.257261:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768046.257263:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768046.257266:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768046.257268:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768046.257270:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768046.257272:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768046.257275:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768046.257275:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768046.257278:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768046.257280:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768046.321278:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768046.321285:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768046.321291:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768046.321295:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768046.321298:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768046.321302:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768046.321305:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768046.321307:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768046.321309:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768046.321312:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768046.321315:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768046.321316:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768046.321317:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768046.321318:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768046.321319:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768046.321320:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768046.321321:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768046.321323:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768046.321325:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768046.321326:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768046.321327:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768046.321328:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768046.321329:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768046.321330:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768046.321331:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768046.321332:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768046.321334:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768046.321335:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768046.321336:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768046.321337:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768046.321338:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768046.321339:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768046.321340:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768046.321341:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768046.321358:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768046.321360:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768046.321362:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768046.385249:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768046.385254:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768046.385257:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768046.385261:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768046.385264:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768046.385267:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768046.385269:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768046.385271:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768046.385273:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768047.281247:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768047.281252:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768047.281253:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768047.281254:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768047.281256:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768047.281258:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768047.281260:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768047.281262:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768047.281263:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768047.281263:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768047.281268:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:2.0:1775768047.281268:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768047.281272:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768047.281275:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768047.281275:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1775768047.281275:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768047.281280:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768047.281280:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768047.281281:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768047.281283:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768047.281284:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768047.281285:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768047.281290:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768047.281293:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768047.281295:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768047.281297:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768047.281300:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768047.345324:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768047.345328:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768047.345332:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768047.345333:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768047.345335:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768047.345343:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768047.345346:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768047.345347:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768047.345349:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768047.345350:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768047.345364:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768047.345366:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768047.345368:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768047.409226:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768047.409232:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768047.409234:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768047.409238:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768047.409242:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768047.409245:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768047.409247:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768047.409249:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768047.409251:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768048.305235:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768048.305240:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768048.305242:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768048.305245:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768048.305246:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768048.305248:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768048.305255:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:2.0:1775768048.305255:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768048.305261:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768048.305261:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768048.305267:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768048.305268:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768048.305272:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768048.305276:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768048.305280:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768048.305280:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768048.305284:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768048.305287:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768048.305365:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768048.305368:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768048.305370:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768048.305373:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775768048.305375:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768048.305378:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768048.305380:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768048.305381:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768048.305383:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768048.369264:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768048.369270:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768048.369277:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768048.369278:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768048.369282:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768048.369283:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768048.369287:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768048.369288:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768048.369291:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768048.369293:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768048.369340:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768048.369343:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768048.369346:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768048.433261:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768048.433267:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768048.433268:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768048.433273:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768048.433276:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768048.433279:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768048.433281:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768048.433283:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768048.433285:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768049.329265:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768049.329268:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768049.329273:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768049.329273:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768049.329275:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768049.329276:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768049.329281:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775768049.329282:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775768049.329282:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768049.329289:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768049.329289:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1775768049.329291:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775768049.329293:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768049.329293:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768049.329295:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1775768049.329298:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1775768049.329300:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775768049.329301:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768049.329306:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768049.329307:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:3.0:1775768049.329307:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768049.329308:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768049.329313:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768049.329317:0:13471:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5301 00000400:00000200:0.0:1775768049.329328:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768049.329330:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768049.329334:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:0.0:1775768049.329337:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:2.0:1775768049.329337:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768049.329342:0:13471:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775768049.329345:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000040:0.0:1775768049.329351:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768049.329351:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768049.329354:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768049.329356:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775768049.329359:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768049.329362:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768049.329363:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768049.329367:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768049.329368:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768049.329383:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768049.329386:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:0.0:1775768049.329390:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768049.329394:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:0.0:1775768049.329397:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:0.0:1775768049.329400:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:0.0:1775768049.329404:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768049.329408:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768049.329411:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768049.329413:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768049.329428:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:0.0:1775768049.329433:0:13471:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768049.329436:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768049.329440:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768049.329444:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768049.329448:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768049.329451:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775768049.329453:0:13471:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768049.329461:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fe300. 00000020:00000040:0.0:1775768049.329464:0:13471:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:0.0:1775768049.329469:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768049.329471:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768049.329475:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768049.329478:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768049.329481:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768049.329488:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220029800. 00000100:00000001:0.0:1775768049.329494:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768049.329502:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fe300 x1862022294267520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768049.329511:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768049.329515:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768049.329520:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008a1e3639] to pc [ptlrpcd_rcv+-1] req@ffff9942099fe300 x1862022294267520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768049.329536:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768049.329538:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768049.329542:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775768049.329543:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768049.329545:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768049.329549:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775768049.329551:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775768049.329554:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1775768049.329556:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5301 00000100:00000001:1.0:1775768049.329558:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000400:00000200:0.0:1775768049.329561:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768049.329566:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768049.329568:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775768049.329569:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fe300 x1862022294267520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775768049.329573:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768049.329578:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768049.329581:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768049.329585:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768049.329590:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:1.0:1775768049.329592:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000040:0.0:1775768049.329593:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:0.0:1775768049.329596:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:1.0:1775768049.329596:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1775768049.329599:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 02000000:00000001:1.0:1775768049.329601:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:0.0:1775768049.329602:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 02000000:00000001:1.0:1775768049.329605:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1775768049.329606:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:0.0:1775768049.329611:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 02000000:00000001:1.0:1775768049.329612:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768049.329613:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768049.329619:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00100000:1.0:1775768049.329620:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294267520:192.168.202.156@tcp:250:kworker.0 00000100:00000001:0.0:1775768049.329621:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768049.329626:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00000001:1.0:1775768049.329628:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00080000:0.0:1775768049.329630:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768049.329632:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768049.329633:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768049.329634:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775768049.329636:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768049.329640:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768049.329640:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1775768049.329643:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775768049.329644:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543c800. 00000100:00000001:0.0:1775768049.329645:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:1.0:1775768049.329649:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1775768049.329652:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775768049.329657:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768049.329657:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768049.329661:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775768049.329664:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000100:00000010:0.0:1775768049.329667:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fd880. 00000020:00000040:0.0:1775768049.329672:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000400:00000010:1.0:1775768049.329673:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 02000000:00000001:0.0:1775768049.329675:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768049.329678:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775768049.329681:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294267520, portal 25 00000100:00000001:0.0:1775768049.329682:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768049.329685:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1775768049.329686:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 02000000:00000001:0.0:1775768049.329688:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768049.329689:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641205504 : -112966068346112 : ffff9942099fe300) 02000000:00000010:0.0:1775768049.329696:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220028400. 00000100:00000040:1.0:1775768049.329699:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fe300 x1862022294267520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768074 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768049.329702:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768049.329714:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fd880 x1862022294267648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775768049.329714:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768049.329721:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768049.329726:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000001:0.0:1775768049.329732:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000200:1.0:1775768049.329732:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294267520, offset 0 00000100:00000001:0.0:1775768049.329737:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768049.329741:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00000040:0.0:1775768049.329744:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007f8a5f3b] to pc [ptlrpcd_rcv+-1] req@ffff9942099fd880 x1862022294267648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775768049.329751:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768049.329759:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768049.329760:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00080000:0.0:1775768049.329763:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768049.329766:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000200:1.0:1775768049.329767:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1775768049.329770:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768049.329772:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1775768049.329773:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000100:00000001:0.0:1775768049.329778:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:1.0:1775768049.329780:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000100:00080000:0.0:1775768049.329783:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5301 00000400:00000200:1.0:1775768049.329787:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768049.329792:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225263:225263:256:4294967295] 192.168.202.156@tcp LPNI seq info [225259:225259:8:4294967295] 00000400:00000200:1.0:1775768049.329795:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768049.329802:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768049.329806:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768049.329808:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000800:00000010:1.0:1775768049.329812:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000400:00000200:0.0:1775768049.329813:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768049.329817:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000800:00000200:1.0:1775768049.329822:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000040:0.0:1775768049.329823:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000800:00000200:1.0:1775768049.329826:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000100:00000001:0.0:1775768049.329828:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768049.329834:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000800:00000200:1.1:1775768049.329836:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:0.0:1775768049.329840:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000800:00000200:1.1:1775768049.329840:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000020:00000001:0.0:1775768049.329846:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768049.329850:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000800:00000100:1.0:1775768049.329850:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768049.329853:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000020:00000001:0.0:1775768049.329854:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768049.329857:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000400:00000200:1.0:1775768049.329858:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000001:0.0:1775768049.329862:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:0.0:1775768049.329868:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000400:00000200:1.0:1775768049.329868:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000100:00000040:0.0:1775768049.329876:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000200:1.0:1775768049.329877:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000100:00000001:0.0:1775768049.329879:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:1.0:1775768049.329881:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000040:0.0:1775768049.329883:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768049.329888:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:1.0:1775768049.329888:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000020:00000040:0.0:1775768049.329893:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:0.0:1775768049.329896:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768049.329898:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768049.329902:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000200:1.0:1775768049.329903:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fe300 x1862022294267520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768074 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1775768049.329905:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768049.329911:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768049.329916:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775768049.329920:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775768049.329924:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768049.329924:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1775768049.329928:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768049.329932:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fe300 x1862022294267520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768074 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1775768049.329933:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ff480. 00000020:00000040:0.0:1775768049.329938:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000100:00000001:1.0:1775768049.329945:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768049.329946:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1775768049.329948:0:2399:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:0.0:1775768049.329949:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768049.329953:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768049.329954:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1775768049.329956:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768049.329957:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768049.329962:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000010:0.0:1775768049.329964:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002b800. 00000100:00000001:1.0:1775768049.329964:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299975485 00000100:00000010:1.0:1775768049.329971:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba600. 00000100:00000001:0.0:1775768049.329972:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768049.329979:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00100000:0.0:1775768049.329981:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ff480 x1862022294267776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775768049.329987:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299975485 00000100:00000001:0.0:1775768049.329989:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768049.329992:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775768049.329994:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba540. 00000100:00000040:0.0:1775768049.329996:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e73dcd12] to pc [ptlrpcd_rcv+-1] req@ffff9942099ff480 x1862022294267776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768049.330000:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775768049.330007:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768049.330008:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768049.330013:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775768049.330014:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768049.330020:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775768049.330021:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1775768049.330022:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768049.330025:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768049.330027:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00100000:1.0:1775768049.330030:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768049/real 1775768049] req@ffff9942099fe300 x1862022294267520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768049.330031:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5301 00000400:00000200:0.0:1775768049.330038:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768049.330044:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768049.330049:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768049.330053:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768049.330055:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768049.330057:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000001:1.0:1775768049.330057:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1775768049.330061:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000001:1.0:1775768049.330061:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:0.0:1775768049.330065:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000400:00000010:1.0:1775768049.330065:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000020:00000040:0.0:1775768049.330066:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:0.0:1775768049.330069:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768049.330071:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:0.0:1775768049.330074:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000400:00000200:1.0:1775768049.330074:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000020:00000001:0.0:1775768049.330079:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000400:00000010:1.0:1775768049.330079:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000040:0.0:1775768049.330086:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768049.330087:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1775768049.330088:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:1.0:1775768049.330095:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fe300 x1862022294267520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775768049.330101:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768049.330104:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768049.330108:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:0.0:1775768049.330112:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768049.330113:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768049.330114:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768049.330117:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768049.330121:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775768049.330121:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fe300 x1862022294267520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768049.330122:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775768049.330125:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:0.0:1775768049.330130:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768049.330134:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768049.330140:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fd180. 00000100:00000001:1.0:1775768049.330142:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000040:0.0:1775768049.330143:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:0.0:1775768049.330146:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1775768049.330146:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1775768049.330148:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768049.330151:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1775768049.330151:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:0.0:1775768049.330152:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1775768049.330182:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1775768049.330184:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768049.330189:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fe300 x1862022294267520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1775768049.330191:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220028000. 00000100:00000001:0.0:1775768049.330198:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768049.330199:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768049.330207:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fe300 x1862022294267520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768074 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768049.330211:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fd180 x1862022294267904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775768049.330221:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768049.330227:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768049.330230:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768049.330233:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775768049.330233:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768049.330235:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768049.330236:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768049.330238:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768049.330241:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000040:0.0:1775768049.330242:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004ec16d17] to pc [ptlrpcd_rcv+-1] req@ffff9942099fd180 x1862022294267904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768049.330246:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768049.330250:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768049.330257:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fe300 x1862022294267520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768074 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768049.330259:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768049.330264:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768049.330271:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1775768049.330272:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294267520:192.168.202.156@tcp:250:kworker.0 00000100:00000001:0.0:1775768049.330274:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768049.330278:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768049.330279:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768049.330281:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775768049.330282:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00080000:0.0:1775768049.330285:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5301 00000100:00000040:1.0:1775768049.330287:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fe300 x1862022294267520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768074 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775768049.330294:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768049.330302:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:1.0:1775768049.330304:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 00000100:00080000:0.0:1775768049.330307:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 02000000:00000001:1.0:1775768049.330308:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768049.330313:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543c800. 00000100:00000040:0.0:1775768049.330314:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768049.330319:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1775768049.330319:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768049.330324:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000040:0.0:1775768049.330325:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000040:1.0:1775768049.330327:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000100:00000001:0.0:1775768049.330329:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775768049.330332:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 00000020:00000001:0.0:1775768049.330334:0:13471:0:(genops.c:811:class_conn2export()) Process entered 02000000:00000010:1.0:1775768049.330336:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029800. 00000020:00000040:0.0:1775768049.330337:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:0.0:1775768049.330341:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000001:1.0:1775768049.330342:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000020:00000040:0.0:1775768049.330343:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 02000000:00000001:1.0:1775768049.330346:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000020:00000001:0.0:1775768049.330348:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000010:1.0:1775768049.330350:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fe300. 00000020:00000001:0.0:1775768049.330352:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000001:1.0:1775768049.330355:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000040:0.0:1775768049.330358:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768049.330358:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768049.330362:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768049.330368:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775768049.330370:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:0.0:1775768049.330374:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768049.330374:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768049.330378:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:1.0:1775768049.330379:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768049.330382:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000040:0.0:1775768049.330383:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00000001:1.0:1775768049.330385:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00080000:0.0:1775768049.330389:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000040:1.0:1775768049.330393:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fd880 x1862022294267648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768049.330394:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768049.330399:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768049.330403:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768049.330409:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768049.330411:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768049.330413:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775768049.330415:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1775768049.330429:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 02000000:00000001:1.0:1775768049.330431:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768049.330434:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1775768049.330435:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768049.330436:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768049.330440:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768049.330441:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294267648:192.168.202.156@tcp:38:kworker.0 00000100:00000010:0.0:1775768049.330445:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fed80. 00000100:00000001:1.0:1775768049.330445:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768049.330448:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:0.0:1775768049.330449:0:13471:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:1.0:1775768049.330450:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768049.330453:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768049.330454:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1775768049.330456:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768049.330459:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000010:1.0:1775768049.330459:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543c800. 02000000:00000001:0.0:1775768049.330461:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775768049.330463:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768049.330465:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768049.330470:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000010:0.0:1775768049.330471:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220029400. 00000400:00000010:1.0:1775768049.330473:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000100:00000001:0.0:1775768049.330475:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775768049.330477:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000200:1.0:1775768049.330480:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294267648, portal 10 00000100:00100000:0.0:1775768049.330481:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fed80 x1862022294268032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775768049.330485:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768049.330487:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641202816 : -112966068348800 : ffff9942099fd880) 00000100:00000040:1.0:1775768049.330498:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fd880 x1862022294267648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768074 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768049.330505:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768049.330508:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768049.330515:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000c09b4d8] to pc [ptlrpcd_rcv+-1] req@ffff9942099fed80 x1862022294268032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768049.330517:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768049.330526:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000100:00000001:0.0:1775768049.330527:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775768049.330529:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000040:0.0:1775768049.330531:0:13471:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5311) 00000100:00000200:1.0:1775768049.330535:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294267648, offset 0 00000400:00000200:1.0:1775768049.330541:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768049.330547:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768049.330552:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768049.330556:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768049.330560:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768049.330563:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768049.330566:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768049.330570:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225264:225264:256:4294967295] 192.168.202.156@tcp LPNI seq info [225260:225260:8:4294967295] 00000400:00000200:1.0:1775768049.330574:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768049.330580:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768049.330584:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768049.330587:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000800:00000200:1.0:1775768049.330590:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768049.330592:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775768049.330595:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768049.330597:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1775768049.330600:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768049.330601:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768049.330603:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768049.330606:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768049.330608:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768049.330609:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768049.330611:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768049.330613:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fd880 x1862022294267648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768074 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768049.330619:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768049.330622:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fd880 x1862022294267648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768074 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768049.330626:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768049.330627:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768049.330629:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768049.330632:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:1.0:1775768049.330634:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768049.330636:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299974163 00000100:00000001:1.0:1775768049.330638:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299975485 00000100:00000001:1.0:1775768049.330640:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299974163 00000100:00000001:1.0:1775768049.330642:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299975485 00000100:00000001:1.0:1775768049.330645:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768049.330647:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768049.330649:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768049.330652:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768049/real 1775768049] req@ffff9942099fd880 x1862022294267648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768049.330661:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768049.330663:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768049.330664:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768049.330667:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768049.330668:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768049.330671:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768049.330674:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fd880 x1862022294267648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768049.330683:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fd880 x1862022294267648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768049.330690:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768049.330692:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768049.330694:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768049.330695:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768049.330699:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fd880 x1862022294267648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768049.330706:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768049.330709:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fd880 x1862022294267648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768074 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768049.330714:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768049.330716:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768049.330717:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768049.330718:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768049.330719:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768049.330721:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768049.330723:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768049.330725:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768049.330727:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768049.330729:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fd880 x1862022294267648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768074 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768049.330734:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294267648:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768049.330736:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768049.330738:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ff480 x1862022294267776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768049.330742:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768049.330743:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768049.330744:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768049.330746:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768049.330747:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768049.330749:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294267776:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768049.330751:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768049.330752:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768049.330753:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768049.330755:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768049.330757:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543e800. 02000000:00000001:1.0:1775768049.330758:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768049.330761:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768049.330762:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000400:00000010:1.0:1775768049.330764:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000200:1.0:1775768049.330767:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294267776, portal 10 00000100:00000001:1.0:1775768049.330768:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768049.330769:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641209984 : -112966068341632 : ffff9942099ff480) 00000100:00000040:1.0:1775768049.330773:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ff480 x1862022294267776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768074 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768049.330777:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768049.330779:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768049.330780:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768049.330782:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294267776, offset 0 00000400:00000200:1.0:1775768049.330785:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768049.330788:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768049.330790:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768049.330792:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768049.330794:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768049.330796:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768049.330797:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768049.330800:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225265:225265:256:4294967295] 192.168.202.156@tcp LPNI seq info [225261:225261:8:4294967295] 00000400:00000200:1.0:1775768049.330802:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768049.330807:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768049.330810:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768049.330812:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000800:00000200:1.0:1775768049.330815:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768049.330816:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775768049.330819:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768049.330821:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1775768049.330823:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768049.330825:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768049.330827:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768049.330829:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768049.330831:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768049.330832:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768049.330834:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768049.330836:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ff480 x1862022294267776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768074 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768049.330841:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768049.330843:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ff480 x1862022294267776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768074 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768049.330847:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768049.330848:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768049.330850:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768049.330852:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:1.0:1775768049.330854:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768049.330855:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299974155 00000100:00000001:1.0:1775768049.330857:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299975485 00000100:00000001:1.0:1775768049.330859:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299974155 00000100:00000001:1.0:1775768049.330861:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299975485 00000100:00000001:1.0:1775768049.330863:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768049.330865:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768049.330867:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768049.330870:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768049/real 1775768049] req@ffff9942099ff480 x1862022294267776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768049.330878:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768049.330879:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768049.330881:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768049.330883:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768049.330885:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768049.330887:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768049.330891:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ff480 x1862022294267776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768049.330899:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ff480 x1862022294267776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768049.330907:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768049.330908:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768049.330910:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768049.330912:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768049.330915:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ff480 x1862022294267776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768049.330921:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768049.330924:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ff480 x1862022294267776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768074 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768049.330929:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768049.330931:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768049.330932:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768049.330933:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768049.330934:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768049.330935:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768049.330937:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768049.330939:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768049.330940:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768049.330942:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ff480 x1862022294267776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768074 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768049.330948:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294267776:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768049.330950:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768049.330952:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fd180 x1862022294267904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768049.330956:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768049.330957:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768049.330958:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768049.330959:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768049.330961:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768049.330962:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294267904:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768049.330964:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768049.330966:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768049.330967:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768049.330968:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768049.330970:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543d000. 02000000:00000001:1.0:1775768049.330972:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768049.330974:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768049.330976:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000400:00000010:1.0:1775768049.330979:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000200:1.0:1775768049.330983:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294267904, portal 4 00000100:00000001:1.0:1775768049.330985:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768049.330986:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641201024 : -112966068350592 : ffff9942099fd180) 00000100:00000040:1.0:1775768049.330991:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fd180 x1862022294267904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768074 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768049.330998:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768049.331000:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768049.331002:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768049.331005:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294267904, offset 0 00000400:00000200:1.0:1775768049.331009:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768049.331014:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768049.331018:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768049.331021:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768049.331024:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768049.331026:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768049.331029:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768049.331032:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225266:225266:256:4294967295] 192.168.202.156@tcp LPNI seq info [225262:225262:8:4294967295] 00000400:00000200:1.0:1775768049.331036:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768049.331042:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768049.331046:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768049.331049:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000800:00000200:1.0:1775768049.331051:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768049.331053:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775768049.331056:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768049.331058:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1775768049.331061:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768049.331062:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768049.331064:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768049.331066:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768049.331068:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768049.331070:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768049.331072:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768049.331074:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fd180 x1862022294267904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768074 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768049.331080:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768049.331082:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fd180 x1862022294267904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768074 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768049.331086:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768049.331087:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768049.331089:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768049.331092:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:1.0:1775768049.331094:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768049.331095:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299974178 00000100:00000001:1.0:1775768049.331097:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299975485 00000100:00000001:1.0:1775768049.331099:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299974178 00000100:00000001:1.0:1775768049.331102:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299975485 00000100:00000001:1.0:1775768049.331104:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768049.331106:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768049.331108:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768049.331111:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768049/real 1775768049] req@ffff9942099fd180 x1862022294267904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768049.331119:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768049.331121:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768049.331123:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768049.331125:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768049.331127:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768049.331129:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768049.331132:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fd180 x1862022294267904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768049.331142:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fd180 x1862022294267904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768049.331149:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768049.331151:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768049.331153:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768049.331174:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768049.331179:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fd180 x1862022294267904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768049.331184:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768049.331187:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fd180 x1862022294267904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768074 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768049.331192:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768049.331194:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768049.331195:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768049.331196:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768049.331197:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768049.331199:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768049.331201:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768049.331203:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768049.331204:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768049.331207:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fd180 x1862022294267904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768074 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768049.331212:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294267904:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768049.331214:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768049.331215:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768049.331217:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fd880 x1862022294267648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768074 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768049.331221:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768049.331222:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768049.331224:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543c800. 02000000:00000001:1.0:1775768049.331225:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768049.331226:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768049.331227:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:1.0:1775768049.331229:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768049.331230:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028400. 02000000:00000001:1.0:1775768049.331232:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768049.331233:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768049.331234:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fd880. 00000100:00000001:1.0:1775768049.331236:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768049.331237:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768049.331238:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768049.331240:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ff480 x1862022294267776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768074 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768049.331244:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768049.331245:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768049.331246:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543e800. 02000000:00000001:1.0:1775768049.331248:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768049.331248:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768049.331249:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:1.0:1775768049.331250:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768049.331251:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002b800. 02000000:00000001:1.0:1775768049.331252:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768049.331253:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768049.331253:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ff480. 00000100:00000001:1.0:1775768049.331255:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768049.331255:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768049.331256:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768049.331258:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fd180 x1862022294267904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768074 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768049.331262:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768049.331262:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768049.331263:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543d000. 02000000:00000001:1.0:1775768049.331264:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768049.331265:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768049.331265:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:1.0:1775768049.331267:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768049.331268:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028000. 02000000:00000001:1.0:1775768049.331269:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768049.331270:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768049.331271:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fd180. 00000100:00000001:1.0:1775768049.331272:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768049.331273:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768049.331274:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768049.331276:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768049.331277:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768049.331278:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768049.331279:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768049.331280:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768049.331282:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fed80 x1862022294268032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768049.331286:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768049.331287:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768049.331289:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768049.331291:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768049.331293:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768049.331296:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294268032:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768049.331300:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768049.331301:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768049.331303:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768049.331305:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768049.331308:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543d000. 02000000:00000001:1.0:1775768049.331309:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768049.331313:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768049.331316:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000400:00000010:1.0:1775768049.331319:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000200:1.0:1775768049.331322:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294268032, portal 4 00000100:00000001:1.0:1775768049.331324:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768049.331325:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641208192 : -112966068343424 : ffff9942099fed80) 00000100:00000040:1.0:1775768049.331332:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fed80 x1862022294268032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768074 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768049.331339:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768049.331342:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768049.331345:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768049.331347:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294268032, offset 0 00000400:00000200:1.0:1775768049.331351:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768049.331354:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768049.331357:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768049.331359:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768049.331361:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768049.331363:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768049.331366:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768049.331369:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225267:225267:256:4294967295] 192.168.202.156@tcp LPNI seq info [225263:225263:8:4294967295] 00000400:00000200:1.0:1775768049.331374:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768049.331380:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768049.331385:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768049.331388:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000800:00000200:1.0:1775768049.331392:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768049.331395:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775768049.331400:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768049.331402:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1775768049.331406:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768049.331409:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768049.331411:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768049.331415:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768049.331431:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768049.331433:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768049.331435:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768049.331438:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fed80 x1862022294268032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768074 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768049.331443:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768049.331445:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fed80 x1862022294268032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768074 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768049.331453:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768049.331454:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768049.331457:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768049.331459:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:1.0:1775768049.331462:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768049.331463:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299974182 00000100:00000001:1.0:1775768049.331465:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299975485 00000100:00000001:1.0:1775768049.331467:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299974182 00000100:00000001:1.0:1775768049.331468:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299975485 00000100:00000001:1.0:1775768049.331469:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768049.331471:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768049.331472:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768049.331474:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768049/real 1775768049] req@ffff9942099fed80 x1862022294268032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768049.331479:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768049.331480:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768049.331481:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768049.331483:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768049.331484:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768049.331486:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768049.331488:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fed80 x1862022294268032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768049.331493:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fed80 x1862022294268032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768049.331496:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768049.331497:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768049.331498:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768049.331499:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768049.331501:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fed80 x1862022294268032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768049.331508:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768049.331512:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fed80 x1862022294268032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768074 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768049.331519:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768049.331521:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768049.331523:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768049.331524:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768049.331526:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768049.331529:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768049.331531:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768049.331535:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768049.331538:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768049.331542:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fed80 x1862022294268032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768074 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768049.331550:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294268032:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768049.331554:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768049.331556:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768049.331559:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fed80 x1862022294268032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768074 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768049.331566:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768049.331567:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768049.331569:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543d000. 02000000:00000001:1.0:1775768049.331572:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768049.331573:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768049.331575:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:1.0:1775768049.331577:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768049.331578:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029400. 02000000:00000001:1.0:1775768049.331580:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768049.331582:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768049.331583:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fed80. 00000100:00000001:1.0:1775768049.331584:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768049.331585:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768049.331587:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768049.331589:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768049.331590:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768049.331591:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768049.331592:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768049.393268:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768049.393273:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768049.393277:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768049.393278:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768049.393280:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768049.393281:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768049.393284:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768049.393284:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768049.393286:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768049.393287:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768049.393319:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768049.393320:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768049.393323:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768049.457248:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768049.457254:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768049.457256:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768049.457261:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768049.457265:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768049.457268:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768049.457270:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768049.457272:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768049.457274:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768050.353230:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768050.353230:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768050.353237:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768050.353238:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768050.353241:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768050.353241:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768050.353249:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775768050.353249:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768050.353255:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768050.353255:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768050.353260:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768050.353261:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768050.353265:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768050.353265:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768050.353268:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768050.353270:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768050.353272:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768050.353273:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00100000:0.0:1775768050.353276:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768050.353277:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768050.353278:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768050.353280:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768050.353283:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768050.353284:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768050.353288:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768050.353290:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768050.353293:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768050.417250:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768050.417254:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768050.417257:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768050.417258:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768050.417261:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768050.417262:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768050.417264:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768050.417265:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768050.417267:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768050.417268:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768050.417292:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768050.417294:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768050.417297:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768050.481253:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768050.481258:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768050.481259:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768050.481263:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768050.481266:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768050.481268:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768050.481270:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768050.481272:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768050.481273:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768050.673261:0:14953:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4299975485, at 4299975488 00000100:00000040:1.0:1775768050.673261:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299975485, at 4299975488 00000100:00000010:1.0:1775768050.673284:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cba540. 00000100:00000010:0.0:1775768050.673285:0:14953:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cba600. 00000100:00000001:0.0:1775768051.377249:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768051.377249:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768051.377249:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768051.377256:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768051.377257:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768051.377257:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768051.377258:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768051.377260:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768051.377260:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768051.377263:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:0.0:1775768051.377266:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775768051.377266:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768051.377267:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768051.377271:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768051.377272:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768051.377272:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768051.377276:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768051.377277:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768051.377277:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768051.377279:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768051.377284:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768051.377285:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768051.377288:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768051.377292:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768051.377292:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768051.377295:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768051.377298:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768051.441218:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768051.441225:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768051.441231:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768051.441232:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768051.441234:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768051.441235:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768051.441237:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768051.441238:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768051.441240:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768051.441241:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768051.441254:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768051.441255:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768051.441257:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768051.505212:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768051.505217:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768051.505219:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768051.505222:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768051.505225:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768051.505227:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768051.505229:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768051.505230:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768051.505232:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768052.401242:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768052.401248:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768052.401250:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768052.401250:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768052.401256:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:2.0:1775768052.401257:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768052.401259:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768052.401263:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775768052.401266:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768052.401269:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768052.401271:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768052.401272:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768052.401274:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768052.401275:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768052.401279:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768052.401283:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768052.401285:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768052.401286:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768052.401286:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768052.401290:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768052.401291:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768052.401294:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768052.401295:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768052.401299:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768052.401302:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768052.401304:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768052.401306:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768052.465256:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768052.465264:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768052.465266:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768052.465270:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768052.465274:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768052.465276:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768052.465279:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768052.465281:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768052.465284:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768052.465285:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768052.465288:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768052.465290:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768052.465292:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768052.465293:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768052.465297:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768052.465298:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768052.465315:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768052.465318:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768052.465320:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768052.529222:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768052.529228:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768052.529229:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768052.529233:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768052.529236:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768052.529246:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768052.529248:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768052.529250:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768052.529253:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768053.425254:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768053.425254:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768053.425260:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768053.425261:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768053.425261:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768053.425265:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768053.425265:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768053.425266:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768053.425269:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768053.425272:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775768053.425273:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775768053.425274:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768053.425279:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768053.425279:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768053.425280:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768053.425284:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768053.425284:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768053.425284:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768053.425288:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768053.425291:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768053.425293:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768053.425293:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768053.425296:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768053.425298:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768053.425300:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768053.425303:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768053.425306:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768053.489246:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768053.489252:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768053.489257:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768053.489259:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768053.489262:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768053.489263:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768053.489266:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768053.489267:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768053.489270:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768053.489271:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768053.489289:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768053.489291:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768053.489293:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768053.553257:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768053.553263:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768053.553265:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768053.553270:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768053.553273:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768053.553277:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768053.553280:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768053.553282:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768053.553285:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768054.449236:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768054.449236:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768054.449241:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768054.449242:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768054.449247:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768054.449248:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768054.449250:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768054.449252:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:2.0:1775768054.449253:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768054.449255:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768054.449258:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768054.449259:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768054.449260:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768054.449262:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768054.449264:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1775768054.449266:0:13471:0:(rw.c:1481:ll_writepages()) Process entered 00000100:00000001:2.0:1775768054.449266:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768054.449269:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:1.0:1775768054.449271:0:13471:0:(file.c:5250:cl_sync_file_range()) Process entered 00000100:00000001:2.0:1775768054.449272:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768054.449274:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00000001:0.0:1775768054.449276:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768054.449280:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768054.449281:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1775768054.449283:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994206aba240. 00000100:00100000:0.0:1775768054.449284:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00080000:3.0:1775768054.449287:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775768054.449288:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000010:1.0:1775768054.449291:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994208a8e000. 00000100:00000001:0.0:1775768054.449292:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775768054.449293:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775768054.449296:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768054.449297:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1775768054.449298:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:1.0:1775768054.449298:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994208a8f600. 00000100:00080000:3.0:1775768054.449300:0:36:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768054.449304:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1775768054.449305:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207cba540. 00000100:00000001:3.0:1775768054.449305:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775768054.449312:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5306 00000008:00000010:1.0:1775768054.449313:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207e02ca0. 00020000:00000010:1.0:1775768054.449319:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff9942031b3480. 00000400:00000200:3.0:1775768054.449319:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775768054.449325:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00008000:00000010:1.0:1775768054.449326:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942054ab170. 00000100:00080000:3.0:1775768054.449327:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775768054.449331:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000080:00000010:1.0:1775768054.449334:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99422030bf18. 00000100:00000001:3.0:1775768054.449334:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775768054.449338:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000080:00000010:1.0:1775768054.449340:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff994205751da8. 00000100:00000001:3.0:1775768054.449340:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775768054.449342:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775768054.449344:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000010:1.0:1775768054.449345:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994208a8ec00. 00000020:00000001:3.0:1775768054.449345:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775768054.449348:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000008:00000010:1.0:1775768054.449351:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff994210709ce0. 00000020:00000001:3.0:1775768054.449351:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:3.0:1775768054.449355:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00020000:00000010:1.0:1775768054.449356:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994205743180. 00000100:00000040:3.0:1775768054.449357:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768054.449359:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00008000:00000010:1.0:1775768054.449361:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994219fec110. 00000100:00000040:3.0:1775768054.449362:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768054.449365:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000080:00000010:1.0:1775768054.449378:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942057e2960. 00000020:00000040:3.0:1775768054.449379:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000020:00000001:1.0:1775768054.449383:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107591643720 : -112966117907896 : ffff994206aba248) 00000100:00080000:3.0:1775768054.449384:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000020:00001000:1.0:1775768054.449387:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff994206aba240 00000100:00000001:3.0:1775768054.449388:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768054.449392:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000020:00000001:1.0:1775768054.449394:0:13471:0:(cl_object.c:151:cl_object_top()) ffff99420791a5f8 -> ffff99420791a5f8 02000000:00000001:3.0:1775768054.449394:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1775768054.449398:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 02000000:00000001:3.0:1775768054.449398:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768054.449401:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000080:00000001:1.0:1775768054.449402:0:13471:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000100:00000001:3.0:1775768054.449402:0:36:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000080:00200000:1.0:1775768054.449405:0:13471:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x10176:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000100:00000010:3.0:1775768054.449408:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aad80. 00000020:00000001:1.0:1775768054.449409:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:1.0:1775768054.449412:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000040:3.0:1775768054.449413:0:36:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 00000080:00000001:1.0:1775768054.449415:0:13471:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768054.449417:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00020000:00000002:1.0:1775768054.449419:0:13471:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x10176:0x0]io ffff994205751e08 type 6 ignore/verify layout 1/0 02000000:00000001:3.0:1775768054.449419:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768054.449421:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00020000:00000002:1.0:1775768054.449425:0:13471:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207dfb6c8) owner 0000000000000000/ffff994220080000 02000000:00000001:3.0:1775768054.449425:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00020000:00000001:1.0:1775768054.449430:0:13471:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 02000000:00000001:3.0:1775768054.449431:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775768054.449433:0:13471:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:1.0:1775768054.449436:0:13471:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00020000:00000001:1.0:1775768054.449438:0:13471:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775768054.449438:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2000. 00020000:00000001:1.0:1775768054.449442:0:13471:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:1.0:1775768054.449444:0:13471:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00000100:00000001:3.0:1775768054.449445:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775768054.449446:0:13471:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768054.449449:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:1.0:1775768054.449450:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:1.0:1775768054.449451:0:13471:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00020000:00000002:1.0:1775768054.449453:0:13471:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207dfb6c8) owner 0000000000000000/ffff994220080000 00000100:00100000:3.0:1775768054.449454:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aad80 x1862022294268160/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000001:1.0:1775768054.449456:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768054.449461:0:13471:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:1.0:1775768054.449463:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00020000:00000001:1.0:1775768054.449466:0:13471:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00000100:00000001:3.0:1775768054.449467:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00020000:00200000:1.0:1775768054.449469:0:13471:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00000001:3.0:1775768054.449472:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:1.0:1775768054.449474:0:13471:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:1.0:1775768054.449478:0:13471:0:(lov_io.c:155:lov_sub_get()) Process entered 00000100:00000040:3.0:1775768054.449479:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000045ccf08] to pc [ptlrpcd_rcv+-1] req@ffff9942099aad80 x1862022294268160/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:1.0:1775768054.449480:0:13471:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000020:00000001:1.0:1775768054.449482:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:1.0:1775768054.449486:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994206ababd0. 00000020:00000010:1.0:1775768054.449492:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994201343000. 00000020:00000010:1.0:1775768054.449496:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994201343200. 00000020:00000010:1.0:1775768054.449498:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207cbab40. 00000008:00000010:1.0:1775768054.449501:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207e00000. 00020000:00000010:1.0:1775768054.449504:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff9942031b30c0. 00008000:00000010:1.0:1775768054.449507:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942054a8a98. 00000100:00000001:3.0:1775768054.449509:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:1.0:1775768054.449511:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99422030b0a8. 00000100:00080000:3.0:1775768054.449513:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000080:00000010:1.0:1775768054.449515:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff994205753d98. 00000100:00080000:3.0:1775768054.449518:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000020:00000010:1.0:1775768054.449519:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994201342e00. 00000100:00000001:3.0:1775768054.449521:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000008:00000010:1.0:1775768054.449524:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff994210709080. 00000100:00080000:3.0:1775768054.449524:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00020000:00000010:1.0:1775768054.449529:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff9942057418c0. 00000100:00000001:3.0:1775768054.449530:0:36:0:(import.c:484:import_select_connection()) Process entered 00008000:00000010:1.0:1775768054.449533:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994219fec368. 00000100:00080000:3.0:1775768054.449535:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5306 00000080:00000010:1.0:1775768054.449537:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942057e24b0. 00000020:00000001:1.0:1775768054.449540:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107591646168 : -112966117905448 : ffff994206ababd8) 00000400:00000200:3.0:1775768054.449542:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768054.449543:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00001000:1.0:1775768054.449544:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff994206ababd0 00000100:00000001:0.0:1775768054.449548:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:1.0:1775768054.449548:0:13471:0:(cl_object.c:151:cl_object_top()) ffff99420791ae90 -> ffff99420791a5f8 00000400:00000200:3.0:1775768054.449549:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:0.0:1775768054.449551:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000001:1.0:1775768054.449552:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000100:00080000:3.0:1775768054.449553:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000020:00000001:1.0:1775768054.449556:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000100:00000040:3.0:1775768054.449558:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000001:1.0:1775768054.449559:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00000040:0.0:1775768054.449561:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099aad80 x1862022294268160/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768054.449561:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:1.0:1775768054.449562:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768054.449564:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00020000:00000001:1.0:1775768054.449566:0:13471:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768054.449568:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00020000:00000001:1.0:1775768054.449570:0:13471:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107571233256 : -112966138318360 : ffff9942057431e8) 00000020:00000001:3.0:1775768054.449572:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775768054.449575:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:1.0:1775768054.449576:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000020:00000001:3.0:1775768054.449577:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000001:1.0:1775768054.449580:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775768054.449580:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00020000:00000001:1.0:1775768054.449586:0:13471:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768054.449586:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000001:0.0:1775768054.449587:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768054.449590:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768054.449591:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768054.449591:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 02000000:00000001:0.0:1775768054.449594:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:1.0:1775768054.449596:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 02000000:00000001:0.0:1775768054.449597:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1775768054.449598:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00020000:00000001:1.0:1775768054.449599:0:13471:0:(lov_io.c:1108:lov_io_lock()) Process entered 00020000:00000001:1.0:1775768054.449602:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 02000000:00000001:0.0:1775768054.449603:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768054.449603:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:1.0:1775768054.449606:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00000100:00000040:3.0:1775768054.449606:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000001:1.0:1775768054.449609:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000100:00000001:3.0:1775768054.449610:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775768054.449612:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775768054.449612:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00100000:0.0:1775768054.449614:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099aad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294268160:192.168.202.156@tcp:250:kworker.0 00000020:00000001:1.0:1775768054.449616:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768054.449617:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00020000:00000001:1.0:1775768054.449622:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768054.449622:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768054.449624:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00020000:00000001:1.0:1775768054.449625:0:13471:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768054.449625:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768054.449629:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:1.0:1775768054.449629:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 02000000:00000001:3.0:1775768054.449630:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1775768054.449632:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768054.449633:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768054.449633:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768054.449636:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768054.449638:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1775768054.449638:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000020:00000001:1.0:1775768054.449639:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00020000:00000001:1.0:1775768054.449642:0:13471:0:(lov_io.c:1114:lov_io_start()) Process entered 00000008:00100000:3.0:1775768054.449642:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00020000:00000001:1.0:1775768054.449643:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 02000000:00000010:0.0:1775768054.449645:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002a800. 00000008:00000001:3.0:1775768054.449645:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768054.449647:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00000100:00000001:3.0:1775768054.449648:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768054.449650:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1775768054.449651:0:13471:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000100:00000010:3.0:1775768054.449652:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aa680. 00000008:00000001:1.0:1775768054.449656:0:13471:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000020:00000040:3.0:1775768054.449656:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000400:00000200:0.0:1775768054.449660:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:3.0:1775768054.449661:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000008:00000001:1.0:1775768054.449662:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 02000000:00000001:3.0:1775768054.449663:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775768054.449664:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000100:00000001:3.0:1775768054.449666:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000008:00000020:1.0:1775768054.449667:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 02000000:00000001:3.0:1775768054.449669:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000008:00000001:1.0:1775768054.449670:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1775768054.449672:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000008:00000001:1.0:1775768054.449673:0:13471:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 02000000:00000001:3.0:1775768054.449673:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1775768054.449676:0:13471:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000008:00000001:1.0:1775768054.449679:0:13471:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107754940544 : -112965954611072 : ffff994210675880) 02000000:00000010:3.0:1775768054.449679:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0c00. 00000100:00000200:0.0:1775768054.449680:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294268160, portal 25 00000100:00000001:0.0:1775768054.449684:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775768054.449685:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768054.449686:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640864128 : -112966068687488 : ffff9942099aad80) 00000008:00000020:1.0:1775768054.449690:0:13471:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994210675880 ready 0|-|- wr 1|-|+ rd 0|- 10 in flight 00000100:00000040:0.0:1775768054.449694:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099aad80 x1862022294268160/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768079 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:1.0:1775768054.449696:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:1.0:1775768054.449698:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:1.0:1775768054.449700:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775768054.449701:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aa680 x1862022294268288/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000008:00000001:1.0:1775768054.449705:0:13471:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000008:00000020:1.0:1775768054.449709:0:13471:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994218232fc0 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000100:00000001:0.0:1775768054.449714:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00000001:1.0:1775768054.449719:0:13471:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768054.449721:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000008:00000001:1.0:1775768054.449723:0:13471:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768054.449725:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000010:0.0:1775768054.449726:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000020:00000001:1.0:1775768054.449726:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775768054.449731:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768054.449731:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1775768054.449732:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294268160, offset 0 00020000:00000001:1.0:1775768054.449734:0:13471:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768054.449737:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768054.449740:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00000100:00000040:3.0:1775768054.449740:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006cb1597d] to pc [ptlrpcd_rcv+-1] req@ffff9942099aa680 x1862022294268288/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775768054.449742:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00020000:00000001:1.0:1775768054.449743:0:13471:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00020000:00000001:1.0:1775768054.449746:0:13471:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000020:00000001:1.0:1775768054.449748:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000001:1.0:1775768054.449750:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00020000:00000001:1.0:1775768054.449752:0:13471:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775768054.449755:0:13471:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000100:00000001:3.0:1775768054.449756:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768054.449757:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:1.0:1775768054.449758:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00000100:00080000:3.0:1775768054.449760:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000001:1.0:1775768054.449761:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00020000:00000001:1.0:1775768054.449764:0:13471:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00000100:00080000:3.0:1775768054.449765:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000200:0.0:1775768054.449766:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00020000:00000001:1.0:1775768054.449767:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000001:3.0:1775768054.449768:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000020:00000001:1.0:1775768054.449770:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000100:00080000:3.0:1775768054.449771:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000020:00000001:1.0:1775768054.449773:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000100:00000001:3.0:1775768054.449774:0:36:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:0.0:1775768054.449775:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00020000:00000001:1.0:1775768054.449776:0:13471:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768054.449779:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5306 00020000:00000001:1.0:1775768054.449780:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768054.449783:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00020000:00000001:1.0:1775768054.449783:0:13471:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000400:00000200:3.0:1775768054.449785:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:1.0:1775768054.449786:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:1.0:1775768054.449789:0:13471:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000400:00000200:0.0:1775768054.449791:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000020:00000001:1.0:1775768054.449792:0:13471:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000400:00000200:3.0:1775768054.449792:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000020:00000001:1.0:1775768054.449794:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000400:00000200:0.0:1775768054.449797:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00020000:00000001:1.0:1775768054.449797:0:13471:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00000100:00080000:3.0:1775768054.449797:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00020000:00000001:1.0:1775768054.449800:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:1.0:1775768054.449803:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000001:1.0:1775768054.449805:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000100:00000040:3.0:1775768054.449805:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000200:0.0:1775768054.449806:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225268:225268:256:4294967295] 192.168.202.156@tcp LPNI seq info [225264:225264:8:4294967295] 00020000:00000001:1.0:1775768054.449808:0:13471:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768054.449810:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00020000:00000001:1.0:1775768054.449812:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775768054.449815:0:13471:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000100:00000040:3.0:1775768054.449816:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000200:0.0:1775768054.449817:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:1.0:1775768054.449817:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000100:00000001:3.0:1775768054.449821:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775768054.449822:0:13471:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768054.449827:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:3.0:1775768054.449827:0:36:0:(genops.c:811:class_conn2export()) Process entered 00020000:00000001:1.0:1775768054.449830:0:13471:0:(lov_io.c:675:lov_io_fini()) Process entered 00000020:00000040:3.0:1775768054.449830:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00020000:00000001:1.0:1775768054.449832:0:13471:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000400:00000200:0.0:1775768054.449834:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000020:00000001:1.0:1775768054.449834:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:3.0:1775768054.449834:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775768054.449838:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:1.0:1775768054.449839:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:1.0:1775768054.449841:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff994206ababd0 00000800:00000200:0.0:1775768054.449844:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000020:00000001:3.0:1775768054.449844:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000010:1.0:1775768054.449846:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994201343200. 00000020:00000001:3.0:1775768054.449850:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000800:00000010:0.0:1775768054.449851:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000020:00000010:1.0:1775768054.449853:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207cbab40. 00000100:00000040:3.0:1775768054.449855:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768054.449858:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000008:00000010:1.0:1775768054.449859:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207e00000. 00000100:00000040:3.0:1775768054.449862:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000800:00000200:0.0:1775768054.449863:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00020000:00000010:1.0:1775768054.449865:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff9942031b30c0. 00000100:00000001:3.0:1775768054.449865:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775768054.449869:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000800:00000200:0.0:1775768054.449870:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00008000:00000010:1.0:1775768054.449871:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942054a8a98. 00000100:00080000:3.0:1775768054.449873:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000080:00000010:1.0:1775768054.449876:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99422030b0a8. 00000100:00000001:3.0:1775768054.449877:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768054.449882:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:0.1:1775768054.449882:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000080:00000010:1.0:1775768054.449883:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff994205753d98. 02000000:00000001:3.0:1775768054.449884:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1775768054.449889:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994201343000. 02000000:00000001:3.0:1775768054.449889:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775768054.449889:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000100:00000001:3.0:1775768054.449893:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00000010:1.0:1775768054.449895:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff994210709080. 00000008:00100000:3.0:1775768054.449896:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775768054.449898:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000008:00000001:3.0:1775768054.449900:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:1.0:1775768054.449901:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff9942057418c0. 00000800:00000010:0.0:1775768054.449902:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000100:00000001:3.0:1775768054.449904:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:1.0:1775768054.449906:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994219fec368. 00000400:00000200:0.0:1775768054.449909:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000010:3.0:1775768054.449909:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ab800. 00000080:00000010:1.0:1775768054.449910:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942057e24b0. 00000020:00000040:3.0:1775768054.449913:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000010:1.0:1775768054.449915:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994201342e00. 02000000:00000001:3.0:1775768054.449917:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000200:0.0:1775768054.449918:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 02000000:00000001:3.0:1775768054.449919:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1775768054.449920:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994206ababd0. 00000100:00000001:3.0:1775768054.449922:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:0.0:1775768054.449923:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 02000000:00000001:3.0:1775768054.449924:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00020000:00000001:1.0:1775768054.449925:0:13471:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00000400:00000010:0.0:1775768054.449928:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00020000:00000001:1.0:1775768054.449928:0:13471:0:(lov_io.c:694:lov_io_fini()) Process leaving 02000000:00000001:3.0:1775768054.449928:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1775768054.449932:0:13471:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000100:00000001:0.0:1775768054.449934:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000080:00200000:1.0:1775768054.449934:0:13471:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x10176:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 02000000:00000010:3.0:1775768054.449935:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b1000. 00000100:00000200:0.0:1775768054.449941:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099aad80 x1862022294268160/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768079 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000001:1.0:1775768054.449941:0:13471:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000100:00000001:3.0:1775768054.449942:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768054.449944:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:1.0:1775768054.449949:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff994206aba240 00000100:00100000:3.0:1775768054.449952:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ab800 x1862022294268416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000010:1.0:1775768054.449953:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994208a8f600. 00000100:00000001:0.0:1775768054.449956:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000020:00000010:1.0:1775768054.449961:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207cba540. 00000100:00000040:0.0:1775768054.449962:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099aad80 x1862022294268160/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768079 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000010:1.0:1775768054.449967:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207e02ca0. 00000100:00000001:3.0:1775768054.449970:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00020000:00000010:1.0:1775768054.449971:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff9942031b3480. 00000100:00000001:3.0:1775768054.449974:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:1.0:1775768054.449976:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942054ab170. 00000080:00000010:1.0:1775768054.449981:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99422030bf18. 00000100:00000040:3.0:1775768054.449982:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000089df80c0] to pc [ptlrpcd_rcv+-1] req@ffff9942099ab800 x1862022294268416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768054.449983:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:1.0:1775768054.449984:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff994205751da8. 00000100:00000001:0.0:1775768054.449987:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000010:1.0:1775768054.449989:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994208a8e000. 00000100:00000001:0.0:1775768054.449993:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:1.0:1775768054.449995:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff994210709ce0. 00020000:00000010:1.0:1775768054.450000:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994205743180. 00000400:00000200:0.0:1775768054.450001:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775768054.450002:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:1.0:1775768054.450003:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994219fec110. 00000100:00000001:0.0:1775768054.450005:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299980605 00000100:00080000:3.0:1775768054.450007:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000080:00000010:1.0:1775768054.450008:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942057e2960. 00000100:00000010:0.0:1775768054.450012:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942078a3d80. 00000020:00000010:1.0:1775768054.450012:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994208a8ec00. 00000020:00000010:1.0:1775768054.450015:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994206aba240. 00000080:00000001:1.0:1775768054.450019:0:13471:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768054.450020:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00080000:3.0:1775768054.450020:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000080:00000001:1.0:1775768054.450021:0:13471:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768054.450024:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775768054.450027:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768054.450028:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299980605 00000100:00000001:3.0:1775768054.450031:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00000010:0.0:1775768054.450033:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942078a3300. 00000100:00080000:3.0:1775768054.450036:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5306 00000100:00000001:0.0:1775768054.450038:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775768054.450043:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1775768054.450044:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768054.450048:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775768054.450050:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:0.0:1775768054.450052:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00080000:3.0:1775768054.450053:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775768054.450058:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768054/real 1775768054] req@ffff9942099aad80 x1862022294268160/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775768054.450059:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768054.450063:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775768054.450068:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768054.450072:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775768054.450076:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775768054.450078:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000400:00000001:0.0:1775768054.450079:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:3.0:1775768054.450080:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000001:0.0:1775768054.450082:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000040:3.0:1775768054.450083:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000400:00000010:0.0:1775768054.450085:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000020:00000001:3.0:1775768054.450086:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000400:00000200:0.0:1775768054.450089:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000020:00000001:3.0:1775768054.450090:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000400:00000010:0.0:1775768054.450092:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768054.450097:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:3.0:1775768054.450097:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768054.450100:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:0.0:1775768054.450102:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099aad80 x1862022294268160/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775768054.450104:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768054.450109:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775768054.450114:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00000200:0.0:1775768054.450118:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099aad80 x1862022294268160/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768054.450119:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775768054.450124:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768054.450129:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1775768054.450132:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:3.0:1775768054.450133:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768054.450134:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1775768054.450138:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768054.450139:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775768054.450141:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775768054.450142:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:3.0:1775768054.450145:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000100:00100000:0.0:1775768054.450149:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099aad80 x1862022294268160/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1775768054.450149:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768054.450150:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775768054.450172:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aa300. 00000020:00000040:3.0:1775768054.450175:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:3.0:1775768054.450176:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775768054.450177:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768054.450179:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775768054.450181:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775768054.450184:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768054.450188:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:3.0:1775768054.450190:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b3400. 00000100:00000001:3.0:1775768054.450193:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768054.450196:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099aad80 x1862022294268160/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768079 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768054.450199:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aa300 x1862022294268544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775768054.450218:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768054.450219:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775768054.450220:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768054.450224:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775768054.450225:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f33bdf15] to pc [ptlrpcd_rcv+-1] req@ffff9942099aa300 x1862022294268544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768054.450229:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768054.450232:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768054.450235:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768054.450236:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1775768054.450238:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775768054.450239:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775768054.450240:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775768054.450242:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775768054.450244:0:36:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768054.450245:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1775768054.450247:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775768054.450251:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5306 00000100:00080000:0.0:1775768054.450252:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768054.450255:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000400:00000200:3.0:1775768054.450255:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775768054.450260:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775768054.450261:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768054.450264:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099aad80 x1862022294268160/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768079 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775768054.450265:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768054.450268:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775768054.450271:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768054.450275:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775768054.450278:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775768054.450279:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:3.0:1775768054.450282:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775768054.450284:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000100:00100000:0.0:1775768054.450286:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099aad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294268160:192.168.202.156@tcp:250:kworker.0 00000020:00000001:3.0:1775768054.450286:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:3.0:1775768054.450288:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:3.0:1775768054.450293:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768054.450295:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768054.450295:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775768054.450297:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768054.450299:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1775768054.450299:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775768054.450302:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:3.0:1775768054.450305:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000040:0.0:1775768054.450306:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099aad80 x1862022294268160/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768079 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768054.450307:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768054.450309:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775768054.450311:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775768054.450314:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768054.450317:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775768054.450319:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000100:00000001:0.0:1775768054.450320:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 00000008:00000001:3.0:1775768054.450321:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768054.450323:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768054.450324:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000010:3.0:1775768054.450325:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8a80. 02000000:00000010:0.0:1775768054.450327:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002a800. 00000020:00000040:3.0:1775768054.450327:0:36:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:3.0:1775768054.450331:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775768054.450332:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768054.450333:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1775768054.450334:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775768054.450336:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000020:00000001:0.0:1775768054.450337:0:2399:0:(genops.c:1236:class_import_put()) Process entered 02000000:00000001:3.0:1775768054.450337:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1775768054.450340:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 02000000:00000010:3.0:1775768054.450341:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0400. 00000020:00000001:0.0:1775768054.450343:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1775768054.450344:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768054.450346:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b2000. 00000100:00100000:3.0:1775768054.450348:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8a80 x1862022294268672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1775768054.450350:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768054.450353:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768054.450356:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099aad80. 00000100:00000001:0.0:1775768054.450361:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775768054.450362:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768054.450363:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768054.450364:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768054.450377:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775768054.450379:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006c3265af] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8a80 x1862022294268672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768054.450383:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768054.450385:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768054.450386:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768054.450390:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775768054.450391:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1775768054.450391:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5316) 00000100:00000001:0.0:1775768054.450394:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768054.450400:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099aa680 x1862022294268288/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768054.450408:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768054.450410:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768054.450412:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768054.450413:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768054.450415:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768054.450418:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099aa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294268288:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768054.450422:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768054.450423:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768054.450424:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768054.450426:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768054.450429:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002a800. 02000000:00000001:0.0:1775768054.450431:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768054.450434:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768054.450436:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775768054.450439:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768054.450442:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294268288, portal 10 00000100:00000001:0.0:1775768054.450444:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768054.450445:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640862336 : -112966068689280 : ffff9942099aa680) 00000100:00000040:0.0:1775768054.450451:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099aa680 x1862022294268288/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768079 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768054.450457:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768054.450459:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768054.450461:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775768054.450463:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294268288, offset 0 00000400:00000200:0.0:1775768054.450467:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768054.450470:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768054.450473:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768054.450476:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768054.450479:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768054.450481:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768054.450483:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768054.450487:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225269:225269:256:4294967295] 192.168.202.156@tcp LPNI seq info [225265:225265:8:4294967295] 00000400:00000200:0.0:1775768054.450491:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768054.450497:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768054.450501:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768054.450504:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775768054.450507:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768054.450510:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775768054.450514:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768054.450516:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1775768054.450520:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768054.450522:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775768054.450524:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768054.450527:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768054.450530:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775768054.450532:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775768054.450534:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768054.450537:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099aa680 x1862022294268288/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768079 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768054.450544:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768054.450547:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099aa680 x1862022294268288/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768079 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768054.450553:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768054.450555:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768054.450557:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768054.450560:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:0.0:1775768054.450563:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768054.450565:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299979283 00000100:00000001:0.0:1775768054.450567:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299980605 00000100:00000001:0.0:1775768054.450569:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299979283 00000100:00000001:0.0:1775768054.450571:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299980605 00000100:00000001:0.0:1775768054.450573:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768054.450575:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768054.450577:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768054.450580:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768054/real 1775768054] req@ffff9942099aa680 x1862022294268288/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768054.450587:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768054.450588:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768054.450590:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775768054.450592:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768054.450593:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768054.450595:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768054.450598:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099aa680 x1862022294268288/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768054.450606:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099aa680 x1862022294268288/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768054.450612:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768054.450613:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768054.450615:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768054.450616:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768054.450619:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099aa680 x1862022294268288/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768054.450625:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768054.450628:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099aa680 x1862022294268288/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768079 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768054.450635:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768054.450638:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768054.450639:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768054.450640:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768054.450641:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768054.450644:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768054.450647:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768054.450650:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768054.450652:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768054.450655:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099aa680 x1862022294268288/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768079 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768054.450663:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099aa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294268288:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768054.450667:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768054.450670:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ab800 x1862022294268416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768054.450676:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768054.450677:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768054.450679:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768054.450681:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768054.450682:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768054.450685:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294268416:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768054.450688:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768054.450689:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768054.450691:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768054.450692:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768054.450695:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002bc00. 02000000:00000001:0.0:1775768054.450696:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768054.450699:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768054.450701:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775768054.450703:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768054.450706:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294268416, portal 10 00000100:00000001:0.0:1775768054.450707:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768054.450709:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640866816 : -112966068684800 : ffff9942099ab800) 00000100:00000040:0.0:1775768054.450713:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ab800 x1862022294268416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768079 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768054.450724:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768054.450727:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768054.450728:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775768054.450731:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294268416, offset 0 00000400:00000200:0.0:1775768054.450734:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768054.450737:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768054.450739:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768054.450742:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768054.450744:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768054.450746:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768054.450748:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768054.450752:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225270:225270:256:4294967295] 192.168.202.156@tcp LPNI seq info [225266:225266:8:4294967295] 00000400:00000200:0.0:1775768054.450756:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768054.450762:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768054.450766:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768054.450768:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775768054.450771:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768054.450774:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775768054.450778:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768054.450780:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1775768054.450783:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768054.450785:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775768054.450787:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768054.450790:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768054.450793:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775768054.450794:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775768054.450796:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768054.450799:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ab800 x1862022294268416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768079 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768054.450806:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768054.450809:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ab800 x1862022294268416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768079 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768054.450821:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768054.450822:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768054.450825:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768054.450827:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:0.0:1775768054.450830:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768054.450831:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299979275 00000100:00000001:0.0:1775768054.450833:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299980605 00000100:00000001:0.0:1775768054.450836:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299979275 00000100:00000001:0.0:1775768054.450838:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299980605 00000100:00000001:0.0:1775768054.450840:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768054.450841:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768054.450843:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768054.450846:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768054/real 1775768054] req@ffff9942099ab800 x1862022294268416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768054.450853:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768054.450854:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768054.450855:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775768054.450857:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768054.450859:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768054.450860:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768054.450863:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ab800 x1862022294268416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768054.450870:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ab800 x1862022294268416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768054.450876:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768054.450878:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768054.450879:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768054.450880:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768054.450883:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ab800 x1862022294268416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768054.450890:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768054.450894:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ab800 x1862022294268416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768079 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768054.450900:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768054.450902:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768054.450904:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768054.450905:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768054.450907:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768054.450909:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768054.450911:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768054.450914:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768054.450916:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768054.450920:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ab800 x1862022294268416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768079 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768054.450927:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294268416:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768054.450930:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768054.450933:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099aa300 x1862022294268544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768054.450940:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768054.450941:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768054.450942:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768054.450944:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768054.450946:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768054.450949:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099aa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294268544:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768054.450952:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768054.450953:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768054.450955:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768054.450956:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768054.450959:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220029400. 02000000:00000001:0.0:1775768054.450960:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768054.450963:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768054.450965:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775768054.450968:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768054.450970:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294268544, portal 4 00000100:00000001:0.0:1775768054.450972:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768054.450973:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640861440 : -112966068690176 : ffff9942099aa300) 00000100:00000040:0.0:1775768054.450977:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099aa300 x1862022294268544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768079 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768054.450984:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768054.450986:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768054.450988:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775768054.450990:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294268544, offset 0 00000400:00000200:0.0:1775768054.450994:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768054.450997:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768054.450999:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768054.451001:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768054.451004:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768054.451006:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768054.451008:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768054.451011:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225271:225271:256:4294967295] 192.168.202.156@tcp LPNI seq info [225267:225267:8:4294967295] 00000400:00000200:0.0:1775768054.451015:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768054.451021:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768054.451025:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768054.451027:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775768054.451030:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768054.451033:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775768054.451037:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768054.451039:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1775768054.451043:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768054.451045:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775768054.451047:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768054.451050:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768054.451052:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775768054.451054:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775768054.451055:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768054.451058:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099aa300 x1862022294268544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768079 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768054.451066:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768054.451068:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099aa300 x1862022294268544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768079 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768054.451075:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768054.451076:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768054.451078:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768054.451080:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:0.0:1775768054.451083:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768054.451085:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299979298 00000100:00000001:0.0:1775768054.451087:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299980605 00000100:00000001:0.0:1775768054.451089:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299979298 00000100:00000001:0.0:1775768054.451091:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299980605 00000100:00000001:0.0:1775768054.451093:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768054.451094:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768054.451096:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768054.451099:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768054/real 1775768054] req@ffff9942099aa300 x1862022294268544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768054.451107:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768054.451108:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768054.451109:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775768054.451111:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768054.451113:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768054.451115:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768054.451118:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099aa300 x1862022294268544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768054.451125:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099aa300 x1862022294268544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768054.451131:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768054.451132:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768054.451134:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768054.451135:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768054.451138:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099aa300 x1862022294268544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768054.451144:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768054.451148:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099aa300 x1862022294268544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768079 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768054.451171:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768054.451173:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768054.451175:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768054.451176:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768054.451178:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768054.451181:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768054.451183:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768054.451186:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768054.451188:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768054.451192:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099aa300 x1862022294268544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768079 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768054.451199:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099aa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294268544:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768054.451202:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768054.451205:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a8a80 x1862022294268672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768054.451212:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768054.451213:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768054.451215:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768054.451216:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768054.451218:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768054.451221:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294268672:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768054.451223:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768054.451225:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768054.451227:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768054.451228:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768054.451231:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220028000. 02000000:00000001:0.0:1775768054.451233:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768054.451235:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768054.451237:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775768054.451240:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768054.451242:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294268672, portal 4 00000100:00000001:0.0:1775768054.451244:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768054.451245:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640855168 : -112966068696448 : ffff9942099a8a80) 00000100:00000040:0.0:1775768054.451250:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a8a80 x1862022294268672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768079 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768054.451256:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768054.451258:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768054.451260:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775768054.451262:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294268672, offset 0 00000400:00000200:0.0:1775768054.451265:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768054.451268:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768054.451271:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768054.451273:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768054.451275:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768054.451277:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768054.451279:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768054.451282:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225272:225272:256:4294967295] 192.168.202.156@tcp LPNI seq info [225268:225268:8:4294967295] 00000400:00000200:0.0:1775768054.451286:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768054.451292:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768054.451295:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768054.451298:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775768054.451302:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768054.451304:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775768054.451307:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768054.451309:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1775768054.451313:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768054.451315:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775768054.451317:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768054.451320:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768054.451322:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775768054.451324:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775768054.451326:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768054.451329:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a8a80 x1862022294268672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768079 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768054.451336:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768054.451339:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a8a80 x1862022294268672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768079 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768054.451345:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768054.451346:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768054.451349:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768054.451351:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:0.0:1775768054.451354:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768054.451356:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299979302 00000100:00000001:0.0:1775768054.451358:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299980605 00000100:00000001:0.0:1775768054.451360:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299979302 00000100:00000001:0.0:1775768054.451362:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299980605 00000100:00000001:0.0:1775768054.451364:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768054.451366:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768054.451381:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768054.451384:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768054/real 1775768054] req@ffff9942099a8a80 x1862022294268672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768054.451391:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768054.451392:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768054.451394:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775768054.451396:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768054.451398:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768054.451400:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768054.451403:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a8a80 x1862022294268672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768054.451409:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a8a80 x1862022294268672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768054.451416:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768054.451417:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768054.451419:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768054.451420:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768054.451423:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a8a80 x1862022294268672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768054.451429:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768054.451432:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a8a80 x1862022294268672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768079 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768054.451438:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768054.451441:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768054.451442:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768054.451443:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768054.451445:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768054.451448:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768054.451449:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768054.451453:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768054.451454:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768054.451458:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a8a80 x1862022294268672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768079 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768054.451465:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294268672:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768054.451468:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768054.451470:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768054.451473:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099aa680 x1862022294268288/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768079 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768054.451479:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768054.451480:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768054.451482:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002a800. 02000000:00000001:0.0:1775768054.451484:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768054.451485:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768054.451486:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:0.0:1775768054.451488:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768054.451490:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b0c00. 02000000:00000001:0.0:1775768054.451492:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768054.451494:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768054.451495:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099aa680. 00000100:00000001:0.0:1775768054.451497:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768054.451498:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768054.451500:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768054.451503:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ab800 x1862022294268416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768079 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768054.451508:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768054.451509:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768054.451510:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002bc00. 02000000:00000001:0.0:1775768054.451512:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768054.451513:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768054.451515:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:0.0:1775768054.451516:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768054.451518:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b1000. 02000000:00000001:0.0:1775768054.451520:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768054.451521:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768054.451522:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ab800. 00000100:00000001:0.0:1775768054.451524:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768054.451525:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768054.451526:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768054.451529:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099aa300 x1862022294268544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768079 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768054.451535:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768054.451536:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768054.451538:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220029400. 02000000:00000001:0.0:1775768054.451539:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768054.451540:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768054.451541:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:0.0:1775768054.451542:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768054.451544:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b3400. 02000000:00000001:0.0:1775768054.451546:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768054.451547:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768054.451548:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099aa300. 00000100:00000001:0.0:1775768054.451550:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768054.451551:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768054.451552:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768054.451555:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a8a80 x1862022294268672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768079 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768054.451561:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768054.451562:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768054.451563:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220028000. 02000000:00000001:0.0:1775768054.451564:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768054.451565:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768054.451566:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:0.0:1775768054.451568:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768054.451569:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b0400. 02000000:00000001:0.0:1775768054.451571:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768054.451571:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768054.451573:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a8a80. 00000100:00000001:0.0:1775768054.451575:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768054.451576:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768054.451578:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768054.451580:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768054.451581:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768054.451583:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775768054.451585:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768054.513242:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768054.513247:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768054.513252:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768054.513254:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768054.513257:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768054.513258:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768054.513261:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768054.513262:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768054.513265:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768054.513266:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768054.513284:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768054.513286:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768054.513289:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768054.577226:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768054.577231:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768054.577232:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768054.577237:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768054.577239:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768054.577242:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768054.577244:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768054.577246:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768054.577248:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768055.473251:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768055.473251:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768055.473252:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768055.473257:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768055.473257:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768055.473259:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768055.473260:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768055.473263:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:0.0:1775768055.473265:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775768055.473266:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768055.473266:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768055.473272:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768055.473272:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1775768055.473272:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768055.473277:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768055.473277:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768055.473277:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768055.473282:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768055.473282:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768055.473284:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768055.473287:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768055.473287:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768055.473290:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768055.473292:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768055.473293:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768055.473296:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768055.473298:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768055.537230:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768055.537237:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768055.537243:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768055.537246:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768055.537248:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768055.537251:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768055.537253:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768055.537256:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768055.537258:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768055.537260:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768055.537264:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768055.537265:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768055.537266:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768055.537268:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768055.537269:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768055.537271:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768055.537272:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768055.537281:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768055.537284:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768055.537285:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768055.537286:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768055.537287:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768055.537287:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768055.537288:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768055.537289:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768055.537291:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768055.537292:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768055.537293:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768055.537294:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768055.537294:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768055.537295:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768055.537296:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768055.537297:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768055.537298:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768055.537312:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768055.537314:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768055.537315:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768055.601253:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768055.601261:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768055.601263:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768055.601268:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768055.601270:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768055.601273:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768055.601275:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768055.601277:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768055.601279:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768055.793213:0:14953:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4299980605, at 4299980608 00000100:00000040:1.0:1775768055.793215:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299980605, at 4299980608 00000100:00000010:1.0:1775768055.793229:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3300. 00000100:00000010:0.0:1775768055.793231:0:14953:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3d80. 00000100:00000001:0.0:1775768056.497225:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768056.497225:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768056.497231:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768056.497232:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768056.497233:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768056.497235:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768056.497239:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775768056.497239:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768056.497243:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768056.497244:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768056.497247:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768056.497248:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768056.497249:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768056.497252:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768056.497254:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768056.497255:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768056.497258:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768056.497260:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768056.497410:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768056.497412:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768056.497414:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768056.497416:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768056.497418:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768056.497420:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768056.497421:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768056.497422:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768056.497423:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768056.561240:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768056.561246:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768056.561250:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768056.561252:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768056.561255:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768056.561256:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768056.561260:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768056.561261:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768056.561264:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768056.561265:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768056.561281:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768056.561284:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768056.561286:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768056.625262:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768056.625268:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768056.625270:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768056.625275:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768056.625278:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768056.625291:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768056.625294:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768056.625296:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768056.625299:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768057.521234:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768057.521234:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768057.521234:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768057.521240:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768057.521242:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768057.521242:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768057.521243:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768057.521246:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768057.521246:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768057.521251:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775768057.521251:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:0.0:1775768057.521252:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768057.521256:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768057.521257:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768057.521258:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768057.521261:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768057.521262:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768057.521262:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768057.521265:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768057.521268:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768057.521269:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768057.521274:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768057.521276:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768057.521278:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768057.521280:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768057.521283:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768057.521285:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768057.585251:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768057.585257:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768057.585264:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768057.585265:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768057.585269:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768057.585270:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768057.585273:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768057.585274:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768057.585277:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768057.585278:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768057.585323:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768057.585326:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768057.585329:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768057.649223:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768057.649230:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768057.649232:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768057.649237:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768057.649240:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768057.649244:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768057.649247:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768057.649249:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768057.649252:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768058.545217:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768058.545217:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768058.545217:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768058.545226:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768058.545226:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768058.545227:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768058.545229:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768058.545230:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768058.545230:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768058.545239:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775768058.545239:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:0.0:1775768058.545240:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768058.545243:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768058.545244:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768058.545245:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768058.545250:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768058.545250:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768058.545252:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768058.545256:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768058.545259:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768058.545262:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768058.545262:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768058.545265:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768058.545268:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768058.545268:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768058.545271:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768058.545273:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768058.609255:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768058.609260:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768058.609266:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768058.609267:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768058.609270:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768058.609271:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768058.609274:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768058.609274:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768058.609277:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768058.609278:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768058.609295:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768058.609296:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768058.609298:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768058.673304:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768058.673313:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768058.673315:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768058.673322:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768058.673336:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768058.673342:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768058.673345:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768058.673348:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768058.673351:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768059.569279:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768059.569279:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768059.569284:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768059.569286:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768059.569286:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768059.569289:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768059.569291:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775768059.569294:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768059.569295:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768059.569299:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768059.569302:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768059.569305:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768059.569306:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768059.569310:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768059.569313:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768059.569313:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1775768059.569316:0:36:0:(rw.c:1481:ll_writepages()) Process entered 00000100:00000001:1.0:1775768059.569332:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1775768059.569334:0:36:0:(file.c:5250:cl_sync_file_range()) Process entered 00000100:00000001:1.0:1775768059.569336:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768059.569337:0:36:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:0.0:1775768059.569349:0:36:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff9942103bed80. 00000100:00080000:2.0:1775768059.569354:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000010:0.0:1775768059.569357:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0f200. 00000100:00080000:2.0:1775768059.569361:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000020:00000010:0.0:1775768059.569364:0:36:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210b0f600. 00000100:00000001:2.0:1775768059.569365:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775768059.569369:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000020:00000010:0.0:1775768059.569370:0:36:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942078a3d80. 00000100:00000001:2.0:1775768059.569376:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:3.0:1775768059.569378:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000008:00000010:0.0:1775768059.569380:0:36:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff9942086e6468. 00000100:00080000:2.0:1775768059.569383:0:13471:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5311 00000100:00000001:3.0:1775768059.569383:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768059.569389:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:0.0:1775768059.569391:0:36:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994209592f00. 00000100:00100000:3.0:1775768059.569393:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000400:00000200:2.0:1775768059.569396:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00008000:00000010:0.0:1775768059.569398:0:36:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff99422035b880. 00000100:00000001:3.0:1775768059.569398:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000400:00000200:2.0:1775768059.569405:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:3.0:1775768059.569406:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1775768059.569408:0:13471:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000080:00000010:0.0:1775768059.569409:0:36:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994207e3e150. 00000100:00000001:3.0:1775768059.569409:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000040:2.0:1775768059.569416:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768059.569417:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000080:00000010:0.0:1775768059.569418:0:36:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b2238. 00000100:00000001:2.0:1775768059.569424:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1775768059.569426:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1775768059.569427:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0fc00. 00000100:00000040:2.0:1775768059.569428:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768059.569436:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000008:00000010:0.0:1775768059.569438:0:36:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff99420906be40. 00000020:00000001:2.0:1775768059.569440:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00020000:00000010:0.0:1775768059.569442:0:36:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff9942103ccfc0. 00000020:00000040:2.0:1775768059.569442:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:2.0:1775768059.569450:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00008000:00000010:0.0:1775768059.569451:0:36:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076afa0. 00000080:00000010:0.0:1775768059.569455:0:36:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942107690f0. 00000020:00000040:2.0:1775768059.569455:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:0.0:1775768059.569463:0:36:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107752095112 : -112965957456504 : ffff9942103bed88) 00000020:00001000:0.0:1775768059.569466:0:36:0:(cl_object.c:831:cl_env_get()) 1@ffff9942103bed80 00000020:00000001:2.0:1775768059.569469:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:0.0:1775768059.569470:0:36:0:(cl_object.c:151:cl_object_top()) ffff9942078a76c0 -> ffff9942078a76c0 00000020:00000001:2.0:1775768059.569472:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:0.0:1775768059.569476:0:36:0:(cl_io.c:125:__cl_io_init()) Process entered 00000100:00000040:2.0:1775768059.569477:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768059.569480:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000080:00000001:0.0:1775768059.569481:0:36:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000100:00000040:2.0:1775768059.569484:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000080:00200000:0.0:1775768059.569485:0:36:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000100:00000001:2.0:1775768059.569487:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775768059.569495:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000020:00000001:0.0:1775768059.569497:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1775768059.569499:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00080000:2.0:1775768059.569500:0:13471:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000080:00000001:0.0:1775768059.569502:0:36:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768059.569511:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1775768059.569514:0:36:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017a:0x0]io ffff9942077b2298 type 6 ignore/verify layout 1/0 02000000:00000001:2.0:1775768059.569514:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775768059.569519:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768059.569521:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768059.569522:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775768059.569524:0:13471:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1775768059.569526:0:36:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207c1f9b0) owner 0000000000000000/ffff994201348000 00020000:00000001:0.0:1775768059.569533:0:36:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00000100:00000010:2.0:1775768059.569534:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfce00. 00020000:00000001:0.0:1775768059.569536:0:36:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1775768059.569540:0:36:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00020000:00000001:0.0:1775768059.569541:0:36:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768059.569542:0:36:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1775768059.569545:0:36:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00000020:00000040:2.0:1775768059.569546:0:13471:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 00020000:00000001:0.0:1775768059.569547:0:36:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768059.569552:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1775768059.569554:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1775768059.569556:0:36:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 02000000:00000001:2.0:1775768059.569557:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00020000:00000002:0.0:1775768059.569558:0:36:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207c1f9b0) owner 0000000000000000/ffff994201348000 00000020:00000001:0.0:1775768059.569563:0:36:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768059.569563:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768059.569565:0:36:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:0.0:1775768059.569568:0:36:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000100:00000001:2.0:1775768059.569570:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00020000:00000001:0.0:1775768059.569571:0:36:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00020000:00200000:0.0:1775768059.569575:0:36:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 02000000:00000001:2.0:1775768059.569576:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00020000:00000002:0.0:1775768059.569579:0:36:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:0.0:1775768059.569582:0:36:0:(lov_io.c:155:lov_sub_get()) Process entered 02000000:00000001:2.0:1775768059.569583:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768059.569585:0:36:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000020:00000001:0.0:1775768059.569587:0:36:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:0.0:1775768059.569590:0:36:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff9942103be000. 02000000:00000010:2.0:1775768059.569591:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1e800. 00000020:00000010:0.0:1775768059.569593:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0ec00. 00000020:00000010:0.0:1775768059.569597:0:36:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210b0ea00. 00000100:00000001:2.0:1775768059.569598:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1775768059.569599:0:36:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942078a30c0. 00000008:00000010:0.0:1775768059.569607:0:36:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff9942086e4e18. 00020000:00000010:0.0:1775768059.569610:0:36:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994209592e40. 00000100:00100000:2.0:1775768059.569613:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfce00 x1862022294268800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00008000:00000010:0.0:1775768059.569614:0:36:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994220358000. 00000080:00000010:0.0:1775768059.569622:0:36:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994207e3e3f0. 00000080:00000010:0.0:1775768059.569630:0:36:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b0ff8. 00000100:00000001:2.0:1775768059.569631:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775768059.569636:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1775768059.569638:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0fa00. 00000100:00000040:2.0:1775768059.569642:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e5a7debe] to pc [ptlrpcd_rcv+-1] req@ffff994207dfce00 x1862022294268800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000010:0.0:1775768059.569644:0:36:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff99420906a6e0. 00020000:00000010:0.0:1775768059.569655:0:36:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff9942103cc480. 00008000:00000010:0.0:1775768059.569661:0:36:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076adb0. 00000080:00000010:0.0:1775768059.569665:0:36:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff994210769870. 00000020:00000001:0.0:1775768059.569667:0:36:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107752091656 : -112965957459960 : ffff9942103be008) 00000020:00001000:0.0:1775768059.569670:0:36:0:(cl_object.c:831:cl_env_get()) 1@ffff9942103be000 00000100:00000001:2.0:1775768059.569671:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768059.569673:0:36:0:(cl_object.c:151:cl_object_top()) ffff9942078a7788 -> ffff9942078a76c0 00000100:00080000:2.0:1775768059.569674:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000001:0.0:1775768059.569680:0:36:0:(cl_io.c:125:__cl_io_init()) Process entered 00000100:00080000:2.0:1775768059.569681:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775768059.569684:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000020:00000001:0.0:1775768059.569686:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000100:00080000:2.0:1775768059.569686:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775768059.569690:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775768059.569692:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5311 00000020:00000001:0.0:1775768059.569695:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:0.0:1775768059.569699:0:36:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768059.569699:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768059.569702:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:2.0:1775768059.569704:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00020000:00000001:0.0:1775768059.569705:0:36:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768059.569707:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1775768059.569707:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00020000:00000001:0.0:1775768059.569712:0:36:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107752153128 : -112965957398488 : ffff9942103cd028) 00000100:00000001:1.0:1775768059.569713:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768059.569713:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000001:0.0:1775768059.569721:0:36:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000100:00000001:2.0:1775768059.569721:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:0.0:1775768059.569727:0:36:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768059.569728:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfce00 x1862022294268800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775768059.569730:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00020000:00000001:0.0:1775768059.569739:0:36:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768059.569739:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775768059.569746:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000001:0.0:1775768059.569749:0:36:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768059.569750:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000040:2.0:1775768059.569750:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000100:00000001:1.0:1775768059.569756:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768059.569758:0:36:0:(cl_io.c:314:cl_io_lock()) Process entered 00020000:00000001:0.0:1775768059.569760:0:36:0:(lov_io.c:1108:lov_io_lock()) Process entered 02000000:00000001:1.0:1775768059.569761:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:2.0:1775768059.569761:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00020000:00000001:0.0:1775768059.569762:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 02000000:00000001:1.0:1775768059.569764:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:2.0:1775768059.569766:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:0.0:1775768059.569767:0:36:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:0.0:1775768059.569773:0:36:0:(cl_io.c:286:cl_lockset_lock()) Process entered 02000000:00000001:1.0:1775768059.569773:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768059.569774:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:0.0:1775768059.569777:0:36:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768059.569779:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:0.0:1775768059.569782:0:36:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768059.569782:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294268800:192.168.202.156@tcp:250:kworker.0 00000100:00000040:2.0:1775768059.569787:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00020000:00000001:0.0:1775768059.569795:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768059.569796:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00020000:00000001:0.0:1775768059.569803:0:36:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768059.569804:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768059.569805:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000001:0.0:1775768059.569810:0:36:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000100:00000001:2.0:1775768059.569810:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 02000000:00000001:1.0:1775768059.569812:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:0.0:1775768059.569814:0:36:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775768059.569815:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 02000000:00000001:1.0:1775768059.569816:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768059.569824:0:36:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768059.569825:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:2.0:1775768059.569828:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000020:00000001:0.0:1775768059.569832:0:36:0:(cl_io.c:528:cl_io_start()) Process entered 02000000:00000010:1.0:1775768059.569832:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543e800. 00000100:00000001:2.0:1775768059.569833:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768059.569835:0:36:0:(lov_io.c:1114:lov_io_start()) Process entered 02000000:00000001:1.0:1775768059.569837:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768059.569841:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 02000000:00000001:2.0:1775768059.569842:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000020:00000001:0.0:1775768059.569846:0:36:0:(cl_io.c:528:cl_io_start()) Process entered 00000400:00000200:1.0:1775768059.569847:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:2.0:1775768059.569850:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1775768059.569851:0:36:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000400:00000010:1.0:1775768059.569852:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 02000000:00000001:2.0:1775768059.569856:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1775768059.569858:0:36:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000100:00000001:2.0:1775768059.569859:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000010:1.0:1775768059.569861:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000008:00100000:2.0:1775768059.569864:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775768059.569865:0:36:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000100:00000200:1.0:1775768059.569868:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294268800, portal 25 00000008:00000020:0.0:1775768059.569870:0:36:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:2.0:1775768059.569871:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768059.569873:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000008:00000001:0.0:1775768059.569875:0:36:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768059.569876:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1775768059.569880:0:36:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000100:00000010:2.0:1775768059.569881:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dff480. 00000008:00000001:0.0:1775768059.569882:0:36:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000100:00000001:1.0:1775768059.569882:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611840000 : -112966097711616 : ffff994207dfce00) 00000008:00000001:0.0:1775768059.569885:0:36:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107884691392 : -112965824860224 : ffff994218232fc0) 00000020:00000040:2.0:1775768059.569886:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000008:00000020:0.0:1775768059.569893:0:36:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994218232fc0 ready 0|-|- wr 1|-|+ rd 0|- 10 in flight 02000000:00000001:2.0:1775768059.569893:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775768059.569898:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1775768059.569901:0:36:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000100:00000040:1.0:1775768059.569902:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfce00 x1862022294268800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768084 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768059.569902:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000008:00000020:0.0:1775768059.569907:0:36:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 02000000:00000001:2.0:1775768059.569908:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000008:00000001:0.0:1775768059.569913:0:36:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1775768059.569914:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1775768059.569924:0:36:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 02000000:00000010:2.0:1775768059.569926:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1d400. 00000100:00000001:1.0:1775768059.569930:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00000020:0.0:1775768059.569931:0:36:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994210675880 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000100:00000001:2.0:1775768059.569939:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768059.569941:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000008:00000001:0.0:1775768059.569951:0:36:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:1.0:1775768059.569953:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00100000:2.0:1775768059.569954:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dff480 x1862022294268928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000008:00000001:0.0:1775768059.569958:0:36:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775768059.569962:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294268800, offset 0 00000020:00000001:0.0:1775768059.569966:0:36:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768059.569975:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768059.569981:0:36:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768059.569986:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000020:00000001:0.0:1775768059.569987:0:36:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768059.569995:0:36:0:(cl_io.c:555:cl_io_end()) Process entered 00000100:00000001:2.0:1775768059.569999:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00020000:00000001:0.0:1775768059.570000:0:36:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00020000:00000001:0.0:1775768059.570002:0:36:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000100:00000001:2.0:1775768059.570003:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768059.570007:0:36:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000001:0.0:1775768059.570012:0:36:0:(cl_io.c:563:cl_io_end()) Process leaving 00000400:00000200:1.0:1775768059.570013:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:0.0:1775768059.570015:0:36:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768059.570015:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093a15639] to pc [ptlrpcd_rcv+-1] req@ffff994207dff480 x1862022294268928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:0.0:1775768059.570022:0:36:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000020:00000001:0.0:1775768059.570027:0:36:0:(cl_io.c:563:cl_io_end()) Process leaving 00000400:00000200:1.0:1775768059.570028:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000020:00000001:0.0:1775768059.570032:0:36:0:(cl_io.c:352:cl_io_unlock()) Process entered 00020000:00000001:0.0:1775768059.570037:0:36:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00020000:00000001:0.0:1775768059.570041:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:0.0:1775768059.570044:0:36:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000400:00000200:1.0:1775768059.570047:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:0.0:1775768059.570049:0:36:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:0.0:1775768059.570055:0:36:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768059.570057:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000100:00000001:2.0:1775768059.570057:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768059.570060:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768059.570063:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00020000:00000001:0.0:1775768059.570066:0:36:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000020:00000001:0.0:1775768059.570071:0:36:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000400:00000200:1.0:1775768059.570072:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000100:00080000:2.0:1775768059.570072:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000020:00000001:0.0:1775768059.570074:0:36:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000100:00000001:2.0:1775768059.570080:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000020:00000001:0.0:1775768059.570081:0:36:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000400:00000200:1.0:1775768059.570082:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000020:00000001:0.0:1775768059.570086:0:36:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000100:00080000:2.0:1775768059.570087:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00020000:00000001:0.0:1775768059.570089:0:36:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00020000:00000001:0.0:1775768059.570090:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000001:2.0:1775768059.570092:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000020:00000001:0.0:1775768059.570093:0:36:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000001:0.0:1775768059.570096:0:36:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000400:00000200:1.0:1775768059.570100:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225273:225273:256:4294967295] 192.168.202.156@tcp LPNI seq info [225269:225269:8:4294967295] 00000100:00080000:2.0:1775768059.570100:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5311 00020000:00000001:0.0:1775768059.570101:0:36:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768059.570108:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768059.570113:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:0.0:1775768059.570114:0:36:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000020:00000001:0.0:1775768059.570119:0:36:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000020:00000001:0.0:1775768059.570122:0:36:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768059.570125:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000400:00000200:1.0:1775768059.570129:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:0.0:1775768059.570131:0:36:0:(cl_io.c:71:cl_io_fini()) Process entered 00000100:00080000:2.0:1775768059.570132:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00020000:00000001:0.0:1775768059.570135:0:36:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:0.0:1775768059.570140:0:36:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000100:00000040:2.0:1775768059.570140:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000001:0.0:1775768059.570144:0:36:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:0.0:1775768059.570149:0:36:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000100:00000001:2.0:1775768059.570150:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1775768059.570175:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768059.570181:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768059.570186:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000020:00001000:0.0:1775768059.570189:0:36:0:(cl_object.c:925:cl_env_put()) 1@ffff9942103be000 00000800:00000200:1.0:1775768059.570196:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000010:0.0:1775768059.570197:0:36:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210b0ea00. 00000800:00000200:1.0:1775768059.570202:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000100:00000040:2.0:1775768059.570203:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000010:0.0:1775768059.570207:0:36:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942078a30c0. 00000100:00000001:2.0:1775768059.570209:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000008:00000010:0.0:1775768059.570211:0:36:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff9942086e4e18. 00000800:00000200:1.1:1775768059.570211:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000001:2.0:1775768059.570215:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775768059.570220:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000800:00000200:1.1:1775768059.570221:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00020000:00000010:0.0:1775768059.570222:0:36:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994209592e40. 00000020:00000001:2.0:1775768059.570226:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00008000:00000010:0.0:1775768059.570231:0:36:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994220358000. 00000020:00000040:2.0:1775768059.570232:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000080:00000010:0.0:1775768059.570240:0:36:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994207e3e3f0. 00000800:00000100:1.0:1775768059.570240:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000020:00000001:2.0:1775768059.570241:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000800:00000010:1.0:1775768059.570249:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000020:00000001:2.0:1775768059.570250:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000080:00000010:0.0:1775768059.570251:0:36:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b0ff8. 00000400:00000200:1.0:1775768059.570259:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000010:0.0:1775768059.570260:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0ec00. 00000100:00000040:2.0:1775768059.570262:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000008:00000010:0.0:1775768059.570268:0:36:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff99420906a6e0. 00000100:00000001:2.0:1775768059.570269:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00020000:00000010:0.0:1775768059.570278:0:36:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff9942103cc480. 00000100:00000040:2.0:1775768059.570278:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000200:1.0:1775768059.570281:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00008000:00000010:0.0:1775768059.570285:0:36:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076adb0. 00000100:00000001:2.0:1775768059.570285:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000200:1.0:1775768059.570289:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000020:00000040:2.0:1775768059.570293:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000080:00000010:0.0:1775768059.570294:0:36:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff994210769870. 00000400:00000010:1.0:1775768059.570294:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000020:00000010:0.0:1775768059.570302:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0fa00. 00000100:00080000:2.0:1775768059.570302:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775768059.570305:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000020:00000010:0.0:1775768059.570309:0:36:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff9942103be000. 00000100:00000001:2.0:1775768059.570311:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768059.570314:0:36:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00000100:00000200:1.0:1775768059.570316:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfce00 x1862022294268800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768084 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1775768059.570316:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775768059.570332:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1775768059.570333:0:36:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:0.0:1775768059.570339:0:36:0:(vvp_io.c:280:vvp_io_fini()) Process entered 02000000:00000001:2.0:1775768059.570340:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000080:00200000:0.0:1775768059.570347:0:36:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00000001:2.0:1775768059.570347:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775768059.570351:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000080:00000001:0.0:1775768059.570363:0:36:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000008:00000001:2.0:1775768059.570363:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768059.570365:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000020:00000001:0.0:1775768059.570368:0:36:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000100:00000001:2.0:1775768059.570371:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1775768059.570373:0:36:0:(cl_object.c:925:cl_env_put()) 1@ffff9942103bed80 00000100:00000040:1.0:1775768059.570375:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfce00 x1862022294268800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768084 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1775768059.570375:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfed80. 00000020:00000010:0.0:1775768059.570377:0:36:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210b0f600. 00000020:00000040:2.0:1775768059.570379:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000010:0.0:1775768059.570388:0:36:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942078a3d80. 02000000:00000001:2.0:1775768059.570388:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1775768059.570390:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768059.570391:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768059.570395:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768059.570395:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000008:00000010:0.0:1775768059.570396:0:36:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff9942086e6468. 02000000:00000001:2.0:1775768059.570400:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775768059.570402:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:0.0:1775768059.570403:0:36:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994209592f00. 02000000:00000010:2.0:1775768059.570406:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1c800. 00008000:00000010:0.0:1775768059.570407:0:36:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff99422035b880. 00000100:00000001:1.0:1775768059.570408:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768059.570411:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1775768059.570416:0:36:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994207e3e150. 00000400:00000200:1.0:1775768059.570420:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000080:00000010:0.0:1775768059.570421:0:36:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b2238. 00000100:00100000:2.0:1775768059.570423:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfed80 x1862022294269056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775768059.570430:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299985726 00000020:00000010:0.0:1775768059.570431:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0f200. 00000008:00000010:0.0:1775768059.570439:0:36:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff99420906be40. 00000100:00000010:1.0:1775768059.570441:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba540. 00020000:00000010:0.0:1775768059.570445:0:36:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff9942103ccfc0. 00000100:00000001:1.0:1775768059.570449:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00008000:00000010:0.0:1775768059.570451:0:36:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076afa0. 00000100:00000001:2.0:1775768059.570455:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000080:00000010:0.0:1775768059.570457:0:36:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942107690f0. 00000100:00000001:1.0:1775768059.570457:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299985726 00000100:00000001:2.0:1775768059.570458:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1775768059.570463:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0fc00. 00000100:00000010:1.0:1775768059.570464:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cbab40. 00000020:00000010:0.0:1775768059.570470:0:36:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff9942103bed80. 00000100:00000040:2.0:1775768059.570470:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb9b9eae] to pc [ptlrpcd_rcv+-1] req@ffff994207dfed80 x1862022294269056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768059.570471:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000080:00000001:0.0:1775768059.570479:0:36:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768059.570481:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000001:0.0:1775768059.570487:0:36:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768059.570488:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768059.570492:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768059.570499:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00080000:2.0:1775768059.570500:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000080:00000001:0.0:1775768059.570504:0:36:0:(rw.c:1481:ll_writepages()) Process entered 00000100:00080000:2.0:1775768059.570504:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1775768059.570505:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768059/real 1775768059] req@ffff994207dfce00 x1862022294268800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000001:0.0:1775768059.570509:0:36:0:(file.c:5250:cl_sync_file_range()) Process entered 00000100:00000001:2.0:1775768059.570509:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000020:00000001:0.0:1775768059.570513:0:36:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00080000:2.0:1775768059.570514:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000020:00000010:0.0:1775768059.570519:0:36:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff9942103bed80. 00000100:00000001:2.0:1775768059.570523:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000020:00000010:0.0:1775768059.570527:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0fc00. 00000100:00080000:2.0:1775768059.570530:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5311 00000020:00000010:0.0:1775768059.570536:0:36:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210b0f200. 00000400:00000001:1.0:1775768059.570539:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000010:0.0:1775768059.570543:0:36:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942078a3d80. 00000400:00000001:1.0:1775768059.570543:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1775768059.570544:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00000010:0.0:1775768059.570550:0:36:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff9942086e6468. 00000400:00000010:1.0:1775768059.570551:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:2.0:1775768059.570556:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00020000:00000010:0.0:1775768059.570560:0:36:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994209592f00. 00000400:00000200:1.0:1775768059.570561:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000100:00080000:2.0:1775768059.570563:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00008000:00000010:0.0:1775768059.570565:0:36:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff99422035b880. 00000400:00000010:1.0:1775768059.570567:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000080:00000010:0.0:1775768059.570571:0:36:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994207e3e150. 00000100:00000040:2.0:1775768059.570572:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768059.570573:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000080:00000010:0.0:1775768059.570580:0:36:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b2238. 00000100:00000001:2.0:1775768059.570581:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:1.0:1775768059.570583:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfce00 x1862022294268800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775768059.570586:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000010:0.0:1775768059.570587:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0f600. 00000100:00000001:2.0:1775768059.570592:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000008:00000010:0.0:1775768059.570597:0:36:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff99420906be40. 00000020:00000001:2.0:1775768059.570598:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775768059.570603:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00020000:00000010:0.0:1775768059.570606:0:36:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff9942103ccfc0. 00000020:00000001:2.0:1775768059.570607:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775768059.570613:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00008000:00000010:0.0:1775768059.570614:0:36:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076afa0. 00000020:00000001:2.0:1775768059.570621:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000080:00000010:0.0:1775768059.570622:0:36:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942107690f0. 00000100:00000200:1.0:1775768059.570622:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfce00 x1862022294268800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775768059.570631:0:36:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107752095112 : -112965957456504 : ffff9942103bed88) 00000020:00000001:2.0:1775768059.570632:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00001000:0.0:1775768059.570644:0:36:0:(cl_object.c:831:cl_env_get()) 1@ffff9942103bed80 00000100:00000001:1.0:1775768059.570645:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:0.0:1775768059.570651:0:36:0:(cl_object.c:151:cl_object_top()) ffff9942078a7f58 -> ffff9942078a7f58 00000100:00000001:1.0:1775768059.570651:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768059.570651:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000001:0.0:1775768059.570661:0:36:0:(cl_io.c:125:__cl_io_init()) Process entered 00000100:00000001:1.0:1775768059.570662:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768059.570662:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000080:00000001:0.0:1775768059.570667:0:36:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000100:00000001:1.0:1775768059.570667:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1775768059.570671:0:36:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017b:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000100:00000040:2.0:1775768059.570672:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00100000:1.0:1775768059.570677:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfce00 x1862022294268800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768059.570682:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768059.570684:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1775768059.570690:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000040:2.0:1775768059.570692:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000080:00000001:0.0:1775768059.570697:0:36:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768059.570700:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775768059.570705:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00020000:00000002:0.0:1775768059.570707:0:36:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017b:0x0]io ffff9942077b2298 type 6 ignore/verify layout 1/0 00000100:00000001:2.0:1775768059.570707:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768059.570712:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfce00 x1862022294268800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768084 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1775768059.570716:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775768059.570722:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000002:0.0:1775768059.570723:0:36:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207c1f000) owner 0000000000000000/ffff994201348000 02000000:00000001:2.0:1775768059.570729:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768059.570732:0:36:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00000100:00000001:2.0:1775768059.570736:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00020000:00000001:0.0:1775768059.570738:0:36:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00000008:00100000:2.0:1775768059.570740:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00020000:00000001:0.0:1775768059.570743:0:36:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00020000:00000001:0.0:1775768059.570749:0:36:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1775768059.570750:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768059.570753:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00020000:00000001:0.0:1775768059.570758:0:36:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768059.570759:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768059.570761:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00020000:00000001:0.0:1775768059.570769:0:36:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00000100:00000001:1.0:1775768059.570769:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000010:2.0:1775768059.570770:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfe300. 00020000:00000001:0.0:1775768059.570771:0:36:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768059.570772:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1775768059.570780:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000100:00000001:1.0:1775768059.570781:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000020:00000040:2.0:1775768059.570781:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:0.0:1775768059.570784:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00000001:1.0:1775768059.570785:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1775768059.570785:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00020000:00000001:0.0:1775768059.570786:0:36:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 02000000:00000001:2.0:1775768059.570788:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1775768059.570793:0:36:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207c1f000) owner 0000000000000000/ffff994201348000 00000100:00080000:1.0:1775768059.570793:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:2.0:1775768059.570793:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775768059.570798:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000020:00000001:0.0:1775768059.570802:0:36:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768059.570803:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768059.570807:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000020:00000001:0.0:1775768059.570811:0:36:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:0.0:1775768059.570813:0:36:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000100:00000001:1.0:1775768059.570813:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000010:2.0:1775768059.570813:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1ec00. 00020000:00000001:0.0:1775768059.570816:0:36:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00020000:00200000:0.0:1775768059.570822:0:36:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00000001:2.0:1775768059.570824:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1775768059.570828:0:36:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00000100:00000040:1.0:1775768059.570828:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfce00 x1862022294268800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768084 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768059.570834:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfe300 x1862022294269184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00020000:00000001:0.0:1775768059.570840:0:36:0:(lov_io.c:155:lov_sub_get()) Process entered 00020000:00000001:0.0:1775768059.570845:0:36:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000020:00000001:0.0:1775768059.570849:0:36:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:0.0:1775768059.570855:0:36:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff9942103be000. 00000020:00000010:0.0:1775768059.570865:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0fa00. 00000020:00000010:0.0:1775768059.570875:0:36:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210b0ec00. 00000100:00000001:2.0:1775768059.570875:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00100000:1.0:1775768059.570879:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294268800:192.168.202.156@tcp:250:kworker.0 00000020:00000010:0.0:1775768059.570884:0:36:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942078a30c0. 00000100:00000001:2.0:1775768059.570885:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:0.0:1775768059.570895:0:36:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff9942086e4e18. 00000100:00000001:1.0:1775768059.570897:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768059.570898:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c661fef] to pc [ptlrpcd_rcv+-1] req@ffff994207dfe300 x1862022294269184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000010:0.0:1775768059.570901:0:36:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994209592e40. 00000100:00000001:1.0:1775768059.570905:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00008000:00000010:0.0:1775768059.570910:0:36:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994220358000. 00000100:00000040:1.0:1775768059.570916:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfce00 x1862022294268800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768084 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768059.570918:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1775768059.570919:0:36:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994207e3e3f0. 00000100:00080000:2.0:1775768059.570924:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000080:00000010:0.0:1775768059.570926:0:36:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b0ff8. 00000020:00000010:0.0:1775768059.570935:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0ea00. 00000100:00080000:2.0:1775768059.570936:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1775768059.570941:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000008:00000010:0.0:1775768059.570942:0:36:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff99420906a6e0. 00000100:00080000:2.0:1775768059.570944:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768059.570945:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 00020000:00000010:0.0:1775768059.570949:0:36:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff9942103cc480. 00000100:00000001:2.0:1775768059.570950:0:13471:0:(import.c:484:import_select_connection()) Process entered 02000000:00000001:1.0:1775768059.570951:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00008000:00000010:0.0:1775768059.570957:0:36:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076adb0. 00000100:00080000:2.0:1775768059.570958:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5311 02000000:00000010:1.0:1775768059.570959:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543e800. 00000080:00000010:0.0:1775768059.570966:0:36:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff994210769870. 02000000:00000001:1.0:1775768059.570968:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000400:00000200:2.0:1775768059.570970:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:0.0:1775768059.570973:0:36:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107752091656 : -112965957459960 : ffff9942103be008) 00000020:00000001:1.0:1775768059.570973:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000400:00000200:2.0:1775768059.570976:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000020:00000040:1.0:1775768059.570977:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00001000:0.0:1775768059.570982:0:36:0:(cl_object.c:831:cl_env_get()) 1@ffff9942103be000 00000100:00080000:2.0:1775768059.570982:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000020:00000001:1.0:1775768059.570984:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768059.570987:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1e800. 00000020:00000001:0.0:1775768059.570990:0:36:0:(cl_object.c:151:cl_object_top()) ffff9942078a75f8 -> ffff9942078a7f58 00000100:00000040:2.0:1775768059.570991:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 02000000:00000001:1.0:1775768059.570994:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000020:00000001:0.0:1775768059.570997:0:36:0:(cl_io.c:125:__cl_io_init()) Process entered 00000100:00000001:2.0:1775768059.570997:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1775768059.570998:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000020:00000001:0.0:1775768059.571003:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000100:00000010:1.0:1775768059.571003:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfce00. 00000100:00000040:2.0:1775768059.571003:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000001:0.0:1775768059.571008:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00000001:2.0:1775768059.571010:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:1.0:1775768059.571011:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000020:00000001:0.0:1775768059.571012:0:36:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768059.571013:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:1.0:1775768059.571014:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:2.0:1775768059.571016:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00020000:00000001:0.0:1775768059.571020:0:36:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768059.571021:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:1.0:1775768059.571022:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:2.0:1775768059.571024:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00020000:00000001:0.0:1775768059.571028:0:36:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107752153128 : -112965957398488 : ffff9942103cd028) 00000020:00000001:2.0:1775768059.571030:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:0.0:1775768059.571033:0:36:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000020:00000001:2.0:1775768059.571034:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:0.0:1775768059.571035:0:36:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768059.571036:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00020000:00000001:0.0:1775768059.571040:0:36:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768059.571041:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768059.571042:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768059.571048:0:36:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768059.571049:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775768059.571051:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1775768059.571054:0:36:0:(cl_io.c:314:cl_io_lock()) Process entered 00000100:00000001:1.0:1775768059.571056:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00020000:00000001:0.0:1775768059.571057:0:36:0:(lov_io.c:1108:lov_io_lock()) Process entered 00000100:00000040:2.0:1775768059.571058:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00020000:00000001:0.0:1775768059.571062:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000001:1.0:1775768059.571062:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1775768059.571063:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768059.571064:0:36:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:0.0:1775768059.571069:0:36:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000040:2.0:1775768059.571070:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000020:00000001:0.0:1775768059.571071:0:36:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768059.571071:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dff480 x1862022294268928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775768059.571078:0:36:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768059.571083:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00020000:00000001:0.0:1775768059.571086:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768059.571094:0:36:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768059.571094:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768059.571102:0:36:0:(cl_io.c:286:cl_lockset_lock()) Process entered 02000000:00000001:2.0:1775768059.571103:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000020:00000001:0.0:1775768059.571107:0:36:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768059.571109:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1775768059.571116:0:36:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768059.571116:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768059.571123:0:36:0:(cl_io.c:528:cl_io_start()) Process entered 00000100:00000001:1.0:1775768059.571124:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768059.571124:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00020000:00000001:0.0:1775768059.571128:0:36:0:(lov_io.c:1114:lov_io_start()) Process entered 00000100:00000001:1.0:1775768059.571128:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1775768059.571129:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00020000:00000001:0.0:1775768059.571132:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 02000000:00000001:1.0:1775768059.571135:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:0.0:1775768059.571136:0:36:0:(cl_io.c:528:cl_io_start()) Process entered 00000008:00000001:2.0:1775768059.571138:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768059.571140:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1775768059.571141:0:36:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000008:00000001:0.0:1775768059.571144:0:36:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000100:00000001:2.0:1775768059.571144:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768059.571145:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1775768059.571152:0:36:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000100:00100000:1.0:1775768059.571153:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294268928:192.168.202.156@tcp:38:kworker.0 00000100:00000010:2.0:1775768059.571153:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfc700. 00000008:00000020:0.0:1775768059.571182:0:36:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:0.0:1775768059.571186:0:36:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:2.0:1775768059.571188:0:13471:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00000008:00000001:0.0:1775768059.571196:0:36:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000100:00000001:1.0:1775768059.571197:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000008:00000001:0.0:1775768059.571199:0:36:0:(osc_cache.c:2287:osc_next_obj()) Process entered 02000000:00000001:2.0:1775768059.571200:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000008:00000001:0.0:1775768059.571201:0:36:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107884688256 : -112965824863360 : ffff994218232380) 02000000:00000001:1.0:1775768059.571202:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768059.571205:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768059.571206:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000020:0.0:1775768059.571207:0:36:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994218232380 ready 0|-|- wr 1|-|+ rd 0|- 9 in flight 02000000:00000001:1.0:1775768059.571213:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1775768059.571213:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000008:00000001:0.0:1775768059.571218:0:36:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:0.0:1775768059.571222:0:36:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 02000000:00000010:1.0:1775768059.571222:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543e800. 02000000:00000001:2.0:1775768059.571222:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000008:00000001:0.0:1775768059.571227:0:36:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1775768059.571230:0:36:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 02000000:00000001:1.0:1775768059.571230:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768059.571230:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000020:0.0:1775768059.571232:0:36:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994210674fc0 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 02000000:00000010:2.0:1775768059.571242:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1fc00. 00000400:00000200:1.0:1775768059.571244:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00000001:0.0:1775768059.571245:0:36:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768059.571248:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1775768059.571252:0:36:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775768059.571253:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000020:00000001:0.0:1775768059.571256:0:36:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768059.571257:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfc700 x1862022294269312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00020000:00000001:0.0:1775768059.571261:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775768059.571264:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00020000:00000001:0.0:1775768059.571272:0:36:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775768059.571275:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294268928, portal 10 00000020:00000001:0.0:1775768059.571282:0:36:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768059.571283:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768059.571289:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611849856 : -112966097701760 : ffff994207dff480) 00000020:00000001:0.0:1775768059.571291:0:36:0:(cl_io.c:555:cl_io_end()) Process entered 00020000:00000001:0.0:1775768059.571296:0:36:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00000100:00000001:2.0:1775768059.571298:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00020000:00000001:0.0:1775768059.571300:0:36:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000100:00000040:1.0:1775768059.571301:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dff480 x1862022294268928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768084 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775768059.571303:0:36:0:(cl_io.c:555:cl_io_end()) Process entered 00000100:00000001:2.0:1775768059.571303:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768059.571309:0:36:0:(cl_io.c:563:cl_io_end()) Process leaving 00020000:00000001:0.0:1775768059.571314:0:36:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768059.571314:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a176dd1] to pc [ptlrpcd_rcv+-1] req@ffff994207dfc700 x1862022294269312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:0.0:1775768059.571331:0:36:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000020:00000001:0.0:1775768059.571335:0:36:0:(cl_io.c:563:cl_io_end()) Process leaving 00000100:00000001:1.0:1775768059.571335:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:0.0:1775768059.571341:0:36:0:(cl_io.c:352:cl_io_unlock()) Process entered 00020000:00000001:0.0:1775768059.571344:0:36:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00000100:00000040:1.0:1775768059.571345:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00020000:00000001:0.0:1775768059.571350:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000400:00000010:1.0:1775768059.571351:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000020:00000001:0.0:1775768059.571356:0:36:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000100:00000001:2.0:1775768059.571356:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775768059.571357:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294268928, offset 0 00000020:00000001:0.0:1775768059.571360:0:36:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:0.0:1775768059.571363:0:36:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768059.571365:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768059.571366:0:13471:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5322) 00000400:00000200:1.0:1775768059.571367:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00020000:00000001:0.0:1775768059.571374:0:36:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000020:00000001:0.0:1775768059.571376:0:36:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:0.0:1775768059.571380:0:36:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000400:00000200:1.0:1775768059.571381:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:0.0:1775768059.571385:0:36:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000001:0.0:1775768059.571387:0:36:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:0.0:1775768059.571389:0:36:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00000400:00000200:1.0:1775768059.571390:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00020000:00000001:0.0:1775768059.571393:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:0.0:1775768059.571396:0:36:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000400:00000200:1.0:1775768059.571398:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:0.0:1775768059.571399:0:36:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00020000:00000001:0.0:1775768059.571401:0:36:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768059.571402:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00020000:00000001:0.0:1775768059.571409:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768059.571416:0:36:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000400:00000200:1.0:1775768059.571416:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000020:00000001:0.0:1775768059.571420:0:36:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000020:00000001:0.0:1775768059.571423:0:36:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768059.571424:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000020:00000001:0.0:1775768059.571431:0:36:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1775768059.571434:0:36:0:(lov_io.c:675:lov_io_fini()) Process entered 00000400:00000200:1.0:1775768059.571435:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225274:225274:256:4294967295] 192.168.202.156@tcp LPNI seq info [225270:225270:8:4294967295] 00020000:00000001:0.0:1775768059.571438:0:36:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:0.0:1775768059.571443:0:36:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:0.0:1775768059.571447:0:36:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1775768059.571449:0:36:0:(cl_object.c:925:cl_env_put()) 1@ffff9942103be000 00000400:00000200:1.0:1775768059.571449:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000010:0.0:1775768059.571455:0:36:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210b0ec00. 00000020:00000010:0.0:1775768059.571463:0:36:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942078a30c0. 00000400:00000200:1.0:1775768059.571467:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000008:00000010:0.0:1775768059.571469:0:36:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff9942086e4e18. 00020000:00000010:0.0:1775768059.571478:0:36:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994209592e40. 00000800:00000200:1.0:1775768059.571478:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768059.571485:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00008000:00000010:0.0:1775768059.571486:0:36:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994220358000. 00000080:00000010:0.0:1775768059.571493:0:36:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994207e3e3f0. 00000800:00000200:1.0:1775768059.571498:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000080:00000010:0.0:1775768059.571500:0:36:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b0ff8. 00000800:00000200:1.0:1775768059.571501:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000020:00000010:0.0:1775768059.571503:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0fa00. 00000008:00000010:0.0:1775768059.571510:0:36:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff99420906a6e0. 00020000:00000010:0.0:1775768059.571516:0:36:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff9942103cc480. 00000800:00000200:1.1:1775768059.571516:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768059.571524:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00008000:00000010:0.0:1775768059.571525:0:36:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076adb0. 00000080:00000010:0.0:1775768059.571534:0:36:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff994210769870. 00000020:00000010:0.0:1775768059.571541:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0ea00. 00000800:00000100:1.0:1775768059.571541:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768059.571545:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000020:00000010:0.0:1775768059.571546:0:36:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff9942103be000. 00020000:00000001:0.0:1775768059.571551:0:36:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00000400:00000200:1.0:1775768059.571551:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00020000:00000001:0.0:1775768059.571556:0:36:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:0.0:1775768059.571560:0:36:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000400:00000200:1.0:1775768059.571561:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000080:00200000:0.0:1775768059.571563:0:36:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000400:00000200:1.0:1775768059.571565:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000080:00000001:0.0:1775768059.571572:0:36:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000400:00000010:1.0:1775768059.571573:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000020:00000001:0.0:1775768059.571580:0:36:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000100:00000001:1.0:1775768059.571582:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000020:00001000:0.0:1775768059.571583:0:36:0:(cl_object.c:925:cl_env_put()) 1@ffff9942103bed80 00000020:00000010:0.0:1775768059.571590:0:36:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210b0f200. 00000100:00000200:1.0:1775768059.571591:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dff480 x1862022294268928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768084 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1775768059.571598:0:36:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942078a3d80. 00000008:00000010:0.0:1775768059.571605:0:36:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff9942086e6468. 00020000:00000010:0.0:1775768059.571612:0:36:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994209592f00. 00000100:00000001:1.0:1775768059.571615:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00008000:00000010:0.0:1775768059.571616:0:36:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff99422035b880. 00000100:00000040:1.0:1775768059.571622:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dff480 x1862022294268928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768084 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000010:0.0:1775768059.571623:0:36:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994207e3e150. 00000080:00000010:0.0:1775768059.571631:0:36:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b2238. 00000020:00000010:0.0:1775768059.571638:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0fc00. 00000008:00000010:0.0:1775768059.571644:0:36:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff99420906be40. 00020000:00000010:0.0:1775768059.571652:0:36:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff9942103ccfc0. 00000100:00000001:1.0:1775768059.571653:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:0.0:1775768059.571656:0:36:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076afa0. 00000100:00000001:1.0:1775768059.571657:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000080:00000010:0.0:1775768059.571664:0:36:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942107690f0. 00000100:00000001:1.0:1775768059.571665:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1775768059.571668:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0f600. 00000100:00000040:1.0:1775768059.571670:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000020:00000010:0.0:1775768059.571672:0:36:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff9942103bed80. 00000080:00000001:0.0:1775768059.571679:0:36:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1775768059.571679:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768059.571685:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299984404 00000080:00000001:0.0:1775768059.571686:0:36:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768059.571694:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299985726 00000080:00000001:0.0:1775768059.571696:0:36:0:(rw.c:1481:ll_writepages()) Process entered 00000100:00000001:1.0:1775768059.571698:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299984404 00000080:00000001:0.0:1775768059.571699:0:36:0:(file.c:5250:cl_sync_file_range()) Process entered 00000020:00000001:0.0:1775768059.571703:0:36:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00000001:1.0:1775768059.571703:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299985726 00000100:00000001:1.0:1775768059.571709:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1775768059.571710:0:36:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff9942103bed80. 00000100:00000001:1.0:1775768059.571712:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1775768059.571715:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0f600. 00000100:00000001:1.0:1775768059.571715:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000020:00000010:0.0:1775768059.571720:0:36:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210b0fc00. 00000100:00100000:1.0:1775768059.571724:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768059/real 1775768059] req@ffff994207dff480 x1862022294268928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1775768059.571726:0:36:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942078a3d80. 00000008:00000010:0.0:1775768059.571731:0:36:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff9942086e6468. 00020000:00000010:0.0:1775768059.571737:0:36:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994209592f00. 00008000:00000010:0.0:1775768059.571742:0:36:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff99422035b880. 00000080:00000010:0.0:1775768059.571746:0:36:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994207e3e150. 00000080:00000010:0.0:1775768059.571750:0:36:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b2238. 00000020:00000010:0.0:1775768059.571757:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0f200. 00000008:00000010:0.0:1775768059.571760:0:36:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff99420906be40. 00000400:00000001:1.0:1775768059.571760:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00020000:00000010:0.0:1775768059.571765:0:36:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff9942103ccfc0. 00000400:00000001:1.0:1775768059.571766:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00008000:00000010:0.0:1775768059.571771:0:36:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076afa0. 00000400:00000010:1.0:1775768059.571772:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000080:00000010:0.0:1775768059.571776:0:36:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942107690f0. 00000020:00000001:0.0:1775768059.571778:0:36:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107752095112 : -112965957456504 : ffff9942103bed88) 00000400:00000200:1.0:1775768059.571778:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000020:00001000:0.0:1775768059.571786:0:36:0:(cl_object.c:831:cl_env_get()) 1@ffff9942103bed80 00000400:00000010:1.0:1775768059.571787:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000020:00000001:0.0:1775768059.571789:0:36:0:(cl_object.c:151:cl_object_top()) ffff99420791a850 -> ffff99420791a850 00000020:00000001:0.0:1775768059.571791:0:36:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1775768059.571792:0:36:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000100:00000001:1.0:1775768059.571793:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000080:00200000:0.0:1775768059.571794:0:36:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017f:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1775768059.571798:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1775768059.571799:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1775768059.571800:0:36:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775768059.571803:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dff480 x1862022294268928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000002:0.0:1775768059.571804:0:36:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017f:0x0]io ffff9942077b2298 type 6 ignore/verify layout 1/0 00020000:00000002:0.0:1775768059.571814:0:36:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207dfbd90) owner 0000000000000000/ffff994201348000 00020000:00000001:0.0:1775768059.571820:0:36:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1775768059.571821:0:36:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1775768059.571826:0:36:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00020000:00000001:0.0:1775768059.571829:0:36:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768059.571833:0:36:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1775768059.571837:0:36:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1775768059.571841:0:36:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775768059.571842:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dff480 x1862022294268928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775768059.571846:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1775768059.571848:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1775768059.571849:0:36:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00020000:00000002:0.0:1775768059.571852:0:36:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207dfbd90) owner 0000000000000000/ffff994201348000 00000020:00000001:0.0:1775768059.571855:0:36:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768059.571860:0:36:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:0.0:1775768059.571864:0:36:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00020000:00000001:0.0:1775768059.571867:0:36:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00020000:00200000:0.0:1775768059.571873:0:36:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00000002:0.0:1775768059.571878:0:36:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:0.0:1775768059.571884:0:36:0:(lov_io.c:155:lov_sub_get()) Process entered 00020000:00000001:0.0:1775768059.571885:0:36:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000100:00000001:1.0:1775768059.571885:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:0.0:1775768059.571887:0:36:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:0.0:1775768059.571890:0:36:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff9942103be000. 00000100:00000001:1.0:1775768059.571890:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:0.0:1775768059.571894:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0ea00. 00000100:00000001:1.0:1775768059.571895:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000010:0.0:1775768059.571896:0:36:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210b0fa00. 00000020:00000010:0.0:1775768059.571900:0:36:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942078a30c0. 00000100:00000001:1.0:1775768059.571900:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00000010:0.0:1775768059.571904:0:36:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff9942086e4e18. 00020000:00000010:0.0:1775768059.571907:0:36:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994209592e40. 00008000:00000010:0.0:1775768059.571908:0:36:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994220358000. 00000100:00100000:1.0:1775768059.571911:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dff480 x1862022294268928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000010:0.0:1775768059.571916:0:36:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994207e3e3f0. 00000080:00000010:0.0:1775768059.571924:0:36:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b0ff8. 00000100:00000001:1.0:1775768059.571925:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:0.0:1775768059.571930:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0ec00. 00000100:00000040:1.0:1775768059.571931:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dff480 x1862022294268928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768084 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000010:0.0:1775768059.571936:0:36:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff99420906a6e0. 00000100:00000001:1.0:1775768059.571943:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00020000:00000010:0.0:1775768059.571944:0:36:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff9942103cc480. 00000100:00000001:1.0:1775768059.571949:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00008000:00000010:0.0:1775768059.571950:0:36:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076adb0. 00000080:00000010:0.0:1775768059.571955:0:36:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff994210769870. 00000100:00000001:1.0:1775768059.571955:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768059.571959:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1775768059.571960:0:36:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107752091656 : -112965957459960 : ffff9942103be008) 00000100:00000001:1.0:1775768059.571964:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000020:00001000:0.0:1775768059.571965:0:36:0:(cl_object.c:831:cl_env_get()) 1@ffff9942103be000 00000020:00000001:0.0:1775768059.571969:0:36:0:(cl_object.c:151:cl_object_top()) ffff99420791a210 -> ffff99420791a850 00000100:00000001:1.0:1775768059.571970:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:0.0:1775768059.571971:0:36:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:0.0:1775768059.571975:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1775768059.571978:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:0.0:1775768059.571979:0:36:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768059.571980:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00020000:00000001:0.0:1775768059.571984:0:36:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768059.571987:0:36:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107752153128 : -112965957398488 : ffff9942103cd028) 00000020:00000001:0.0:1775768059.571989:0:36:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000020:00000001:0.0:1775768059.571992:0:36:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768059.571993:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00020000:00000001:0.0:1775768059.571995:0:36:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768059.572000:0:36:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768059.572002:0:36:0:(cl_io.c:314:cl_io_lock()) Process entered 00000100:00000001:1.0:1775768059.572002:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00020000:00000001:0.0:1775768059.572004:0:36:0:(lov_io.c:1108:lov_io_lock()) Process entered 00020000:00000001:0.0:1775768059.572005:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:0.0:1775768059.572006:0:36:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:0.0:1775768059.572009:0:36:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:0.0:1775768059.572011:0:36:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768059.572014:0:36:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768059.572015:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dff480 x1862022294268928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768084 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:0.0:1775768059.572017:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768059.572020:0:36:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768059.572023:0:36:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:0.0:1775768059.572024:0:36:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768059.572028:0:36:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768059.572031:0:36:0:(cl_io.c:528:cl_io_start()) Process entered 00020000:00000001:0.0:1775768059.572032:0:36:0:(lov_io.c:1114:lov_io_start()) Process entered 00020000:00000001:0.0:1775768059.572035:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:0.0:1775768059.572037:0:36:0:(cl_io.c:528:cl_io_start()) Process entered 00000008:00000001:0.0:1775768059.572040:0:36:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000008:00000001:0.0:1775768059.572045:0:36:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000008:00000001:0.0:1775768059.572050:0:36:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:0.0:1775768059.572052:0:36:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:0.0:1775768059.572055:0:36:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1775768059.572059:0:36:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000008:00000001:0.0:1775768059.572062:0:36:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000008:00000001:0.0:1775768059.572064:0:36:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107754938304 : -112965954613312 : ffff994210674fc0) 00000100:00100000:1.0:1775768059.572065:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294268928:192.168.202.156@tcp:38:kworker.0 00000008:00000020:0.0:1775768059.572068:0:36:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994210674fc0 ready 0|-|- wr 1|-|+ rd 0|- 9 in flight 00000008:00000001:0.0:1775768059.572078:0:36:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:0.0:1775768059.572080:0:36:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:0.0:1775768059.572081:0:36:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1775768059.572083:0:36:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000100:00000001:1.0:1775768059.572084:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000008:00000020:0.0:1775768059.572085:0:36:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994218232380 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000008:00000001:0.0:1775768059.572090:0:36:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768059.572091:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfed80 x1862022294269056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:0.0:1775768059.572093:0:36:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768059.572097:0:36:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768059.572102:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768059.572107:0:36:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768059.572114:0:36:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768059.572120:0:36:0:(cl_io.c:555:cl_io_end()) Process entered 00020000:00000001:0.0:1775768059.572122:0:36:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00020000:00000001:0.0:1775768059.572124:0:36:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000020:00000001:0.0:1775768059.572126:0:36:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000001:0.0:1775768059.572129:0:36:0:(cl_io.c:563:cl_io_end()) Process leaving 00020000:00000001:0.0:1775768059.572132:0:36:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768059.572138:0:36:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000020:00000001:0.0:1775768059.572141:0:36:0:(cl_io.c:563:cl_io_end()) Process leaving 00000020:00000001:0.0:1775768059.572143:0:36:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000100:00000001:1.0:1775768059.572144:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00020000:00000001:0.0:1775768059.572146:0:36:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00000100:00000001:1.0:1775768059.572147:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768059.572150:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 02000000:00000001:1.0:1775768059.572152:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:0.0:1775768059.572153:0:36:0:(cl_io.c:352:cl_io_unlock()) Process entered 02000000:00000001:1.0:1775768059.572177:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768059.572179:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768059.572182:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294269056:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768059.572184:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768059.572186:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:0.0:1775768059.572188:0:36:0:(cl_io.c:373:cl_io_unlock()) Process leaving 02000000:00000001:1.0:1775768059.572189:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768059.572190:0:36:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768059.572192:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00020000:00000001:0.0:1775768059.572193:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768059.572197:0:36:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 02000000:00000010:1.0:1775768059.572197:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543c800. 00000020:00000001:0.0:1775768059.572199:0:36:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:0.0:1775768059.572205:0:36:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 02000000:00000001:1.0:1775768059.572205:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768059.572210:0:36:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000001:0.0:1775768059.572212:0:36:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:0.0:1775768059.572213:0:36:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00020000:00000001:0.0:1775768059.572218:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000400:00000200:1.0:1775768059.572218:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:0.0:1775768059.572220:0:36:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000001:0.0:1775768059.572222:0:36:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00020000:00000001:0.0:1775768059.572224:0:36:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768059.572225:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775768059.572225:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00020000:00000001:0.0:1775768059.572232:0:36:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000020:00000001:0.0:1775768059.572233:0:36:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000020:00000001:0.0:1775768059.572234:0:36:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775768059.572235:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000020:00000001:0.0:1775768059.572237:0:36:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1775768059.572238:0:36:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:0.0:1775768059.572241:0:36:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:0.0:1775768059.572243:0:36:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:0.0:1775768059.572244:0:36:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1775768059.572245:0:36:0:(cl_object.c:925:cl_env_put()) 1@ffff9942103be000 00000100:00000200:1.0:1775768059.572245:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294269056, portal 10 00000020:00000010:0.0:1775768059.572248:0:36:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210b0fa00. 00000100:00000001:1.0:1775768059.572253:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000020:00000010:0.0:1775768059.572254:0:36:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942078a30c0. 00000008:00000010:0.0:1775768059.572258:0:36:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff9942086e4e18. 00000100:00000001:1.0:1775768059.572259:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611848064 : -112966097703552 : ffff994207dfed80) 00020000:00000010:0.0:1775768059.572261:0:36:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994209592e40. 00008000:00000010:0.0:1775768059.572266:0:36:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994220358000. 00000080:00000010:0.0:1775768059.572271:0:36:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994207e3e3f0. 00000080:00000010:0.0:1775768059.572275:0:36:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b0ff8. 00000100:00000040:1.0:1775768059.572276:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfed80 x1862022294269056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768084 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1775768059.572279:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0ea00. 00000008:00000010:0.0:1775768059.572287:0:36:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff99420906a6e0. 00020000:00000010:0.0:1775768059.572295:0:36:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff9942103cc480. 00000100:00000001:1.0:1775768059.572298:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00008000:00000010:0.0:1775768059.572299:0:36:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076adb0. 00000080:00000010:0.0:1775768059.572304:0:36:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff994210769870. 00000100:00000040:1.0:1775768059.572305:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000020:00000010:0.0:1775768059.572312:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0ec00. 00000400:00000010:1.0:1775768059.572313:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000020:00000010:0.0:1775768059.572329:0:36:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff9942103be000. 00020000:00000001:0.0:1775768059.572332:0:36:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00000100:00000200:1.0:1775768059.572334:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294269056, offset 0 00020000:00000001:0.0:1775768059.572336:0:36:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:0.0:1775768059.572339:0:36:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000400:00000200:1.0:1775768059.572341:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000080:00200000:0.0:1775768059.572343:0:36:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017f:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000400:00000200:1.0:1775768059.572350:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000080:00000001:0.0:1775768059.572357:0:36:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000400:00000200:1.0:1775768059.572359:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000020:00000001:0.0:1775768059.572360:0:36:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000400:00000200:1.0:1775768059.572363:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00001000:0.0:1775768059.572365:0:36:0:(cl_object.c:925:cl_env_put()) 1@ffff9942103bed80 00000400:00000200:1.0:1775768059.572366:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000020:00000010:0.0:1775768059.572371:0:36:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210b0fc00. 00000400:00000200:1.0:1775768059.572375:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000020:00000010:0.0:1775768059.572377:0:36:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942078a3d80. 00000008:00000010:0.0:1775768059.572383:0:36:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff9942086e6468. 00000400:00000200:1.0:1775768059.572383:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00020000:00000010:0.0:1775768059.572392:0:36:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994209592f00. 00000400:00000200:1.0:1775768059.572400:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225275:225275:256:4294967295] 192.168.202.156@tcp LPNI seq info [225271:225271:8:4294967295] 00008000:00000010:0.0:1775768059.572401:0:36:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff99422035b880. 00000080:00000010:0.0:1775768059.572408:0:36:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994207e3e150. 00000400:00000200:1.0:1775768059.572415:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000080:00000010:0.0:1775768059.572416:0:36:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b2238. 00000020:00000010:0.0:1775768059.572421:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0f600. 00000008:00000010:0.0:1775768059.572431:0:36:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff99420906be40. 00000400:00000200:1.0:1775768059.572434:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00020000:00000010:0.0:1775768059.572436:0:36:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff9942103ccfc0. 00008000:00000010:0.0:1775768059.572448:0:36:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076afa0. 00000800:00000200:1.0:1775768059.572450:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000080:00000010:0.0:1775768059.572453:0:36:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942107690f0. 00000800:00000010:1.0:1775768059.572454:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000020:00000010:0.0:1775768059.572459:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0f200. 00000020:00000010:0.0:1775768059.572463:0:36:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff9942103bed80. 00000800:00000200:1.0:1775768059.572464:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000080:00000001:0.0:1775768059.572471:0:36:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:1.0:1775768059.572472:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000080:00000001:0.0:1775768059.572475:0:36:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775768059.572484:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768059.572487:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000100:1.0:1775768059.572491:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768059.572492:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768059.572495:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768059.572497:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768059.572500:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768059.572502:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768059.572504:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768059.572509:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfed80 x1862022294269056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768084 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768059.572515:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768059.572517:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfed80 x1862022294269056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768084 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768059.572521:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768059.572522:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768059.572524:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768059.572526:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:1.0:1775768059.572527:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768059.572529:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299984397 00000100:00000001:1.0:1775768059.572530:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299985726 00000100:00000001:1.0:1775768059.572532:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299984397 00000100:00000001:1.0:1775768059.572533:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299985726 00000100:00000001:1.0:1775768059.572534:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768059.572536:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768059.572537:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768059.572539:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768059/real 1775768059] req@ffff994207dfed80 x1862022294269056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768059.572544:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768059.572544:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768059.572545:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768059.572546:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768059.572547:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768059.572548:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768059.572550:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfed80 x1862022294269056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768059.572555:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfed80 x1862022294269056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768059.572558:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768059.572559:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768059.572559:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768059.572560:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768059.572562:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfed80 x1862022294269056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768059.572565:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768059.572567:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfed80 x1862022294269056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768084 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768059.572571:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768059.572572:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768059.572573:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768059.572574:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768059.572574:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768059.572577:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768059.572578:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768059.572580:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768059.572581:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768059.572583:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfed80 x1862022294269056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768084 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768059.572588:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294269056:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768059.572590:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768059.572591:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfe300 x1862022294269184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768059.572595:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768059.572596:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768059.572597:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768059.572598:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768059.572599:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768059.572600:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294269184:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768059.572602:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768059.572604:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768059.572605:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768059.572606:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768059.572608:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543d400. 02000000:00000001:1.0:1775768059.572609:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768059.572611:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768059.572612:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000400:00000010:1.0:1775768059.572614:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000200:1.0:1775768059.572616:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294269184, portal 4 00000100:00000001:1.0:1775768059.572617:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768059.572618:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611845376 : -112966097706240 : ffff994207dfe300) 00000100:00000040:1.0:1775768059.572621:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfe300 x1862022294269184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768084 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768059.572624:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768059.572626:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768059.572627:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768059.572628:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294269184, offset 0 00000400:00000200:1.0:1775768059.572630:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768059.572632:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768059.572634:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768059.572636:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768059.572637:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768059.572639:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768059.572640:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768059.572642:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225276:225276:256:4294967295] 192.168.202.156@tcp LPNI seq info [225272:225272:8:4294967295] 00000400:00000200:1.0:1775768059.572644:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768059.572648:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768059.572650:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768059.572651:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000800:00000200:1.0:1775768059.572654:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768059.572655:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775768059.572657:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768059.572659:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000100:1.0:1775768059.572661:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768059.572662:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768059.572664:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768059.572665:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768059.572667:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768059.572668:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768059.572669:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768059.572671:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfe300 x1862022294269184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768084 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768059.572675:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768059.572676:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfe300 x1862022294269184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768084 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768059.572680:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768059.572681:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768059.572682:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768059.572684:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:1.0:1775768059.572686:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768059.572687:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299984420 00000100:00000001:1.0:1775768059.572688:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299985726 00000100:00000001:1.0:1775768059.572690:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299984420 00000100:00000001:1.0:1775768059.572691:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299985726 00000100:00000001:1.0:1775768059.572692:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768059.572693:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768059.572694:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768059.572696:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768059/real 1775768059] req@ffff994207dfe300 x1862022294269184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768059.572700:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768059.572700:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768059.572701:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768059.572702:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768059.572703:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768059.572704:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768059.572706:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfe300 x1862022294269184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768059.572710:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfe300 x1862022294269184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768059.572713:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768059.572714:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768059.572715:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768059.572715:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768059.572717:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfe300 x1862022294269184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768059.572720:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768059.572722:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfe300 x1862022294269184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768084 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768059.572727:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768059.572728:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768059.572729:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768059.572730:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768059.572730:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768059.572732:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768059.572734:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768059.572735:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768059.572736:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768059.572738:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfe300 x1862022294269184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768084 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768059.572742:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294269184:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768059.572744:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768059.572745:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768059.572747:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dff480 x1862022294268928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768084 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768059.572751:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768059.572752:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768059.572753:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543e800. 02000000:00000001:1.0:1775768059.572754:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768059.572755:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768059.572756:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:1.0:1775768059.572757:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768059.572758:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1d400. 02000000:00000001:1.0:1775768059.572761:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768059.572762:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768059.572763:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dff480. 00000100:00000001:1.0:1775768059.572764:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768059.572765:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768059.572766:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768059.572768:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfed80 x1862022294269056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768084 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768059.572772:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768059.572772:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768059.572773:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543c800. 02000000:00000001:1.0:1775768059.572774:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768059.572775:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768059.572775:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:1.0:1775768059.572776:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768059.572777:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1c800. 02000000:00000001:1.0:1775768059.572778:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768059.572778:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768059.572779:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfed80. 00000100:00000001:1.0:1775768059.572780:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768059.572781:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768059.572782:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768059.572783:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfe300 x1862022294269184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768084 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768059.572786:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768059.572787:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768059.572787:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543d400. 02000000:00000001:1.0:1775768059.572788:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768059.572789:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768059.572789:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:1.0:1775768059.572790:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768059.572791:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1ec00. 02000000:00000001:1.0:1775768059.572792:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768059.572793:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768059.572793:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfe300. 00000100:00000001:1.0:1775768059.572794:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768059.572795:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768059.572796:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768059.572799:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768059.572799:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768059.572801:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768059.572802:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768059.572803:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768059.572804:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfc700 x1862022294269312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768059.572808:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768059.572809:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768059.572810:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768059.572811:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768059.572812:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768059.572813:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294269312:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768059.572815:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768059.572816:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768059.572817:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768059.572818:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768059.572819:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543d400. 02000000:00000001:1.0:1775768059.572820:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768059.572822:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768059.572823:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000400:00000010:1.0:1775768059.572825:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000200:1.0:1775768059.572826:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294269312, portal 4 00000100:00000001:1.0:1775768059.572827:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768059.572828:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611838208 : -112966097713408 : ffff994207dfc700) 00000100:00000040:1.0:1775768059.572831:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfc700 x1862022294269312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768084 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768059.572835:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768059.572836:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768059.572837:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768059.572838:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294269312, offset 0 00000400:00000200:1.0:1775768059.572840:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768059.572842:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768059.572844:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768059.572845:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768059.572846:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768059.572848:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768059.572849:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768059.572851:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225277:225277:256:4294967295] 192.168.202.156@tcp LPNI seq info [225273:225273:8:4294967295] 00000400:00000200:1.0:1775768059.572854:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768059.572857:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768059.572859:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768059.572861:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000800:00000200:1.0:1775768059.572864:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768059.572866:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775768059.572870:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768059.572872:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000100:1.0:1775768059.572876:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768059.572877:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768059.572879:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768059.572883:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768059.572885:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768059.572887:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768059.572889:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768059.572892:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfc700 x1862022294269312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768084 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768059.572899:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768059.572901:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfc700 x1862022294269312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768084 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768059.572907:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768059.572909:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768059.572912:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768059.572914:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:1.0:1775768059.572917:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768059.572919:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299984423 00000100:00000001:1.0:1775768059.572921:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299985726 00000100:00000001:1.0:1775768059.572923:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299984423 00000100:00000001:1.0:1775768059.572926:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299985726 00000100:00000001:1.0:1775768059.572928:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768059.572930:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768059.572931:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768059.572934:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768059/real 1775768059] req@ffff994207dfc700 x1862022294269312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768059.572942:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768059.572943:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768059.572944:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768059.572946:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768059.572948:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768059.572950:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768059.572953:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfc700 x1862022294269312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768059.572960:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfc700 x1862022294269312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768059.572966:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768059.572967:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768059.572969:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768059.572970:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768059.572974:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfc700 x1862022294269312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768059.572979:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768059.572982:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfc700 x1862022294269312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768084 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768059.572988:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768059.572990:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768059.572992:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768059.572993:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768059.572995:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768059.572999:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768059.573001:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768059.573004:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768059.573006:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768059.573010:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfc700 x1862022294269312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768084 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768059.573017:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294269312:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768059.573021:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768059.573022:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768059.573025:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfc700 x1862022294269312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768084 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768059.573031:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768059.573032:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768059.573034:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543d400. 02000000:00000001:1.0:1775768059.573036:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768059.573037:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768059.573038:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:1.0:1775768059.573040:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768059.573041:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1fc00. 02000000:00000001:1.0:1775768059.573043:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768059.573044:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768059.573045:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfc700. 00000100:00000001:1.0:1775768059.573048:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768059.573048:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768059.573050:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768059.573052:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768059.573053:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768059.573054:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768059.573055:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768059.633285:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768059.633290:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768059.633294:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768059.633295:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768059.633298:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768059.633299:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768059.633301:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768059.633302:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768059.633305:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768059.633307:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768059.633339:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768059.633342:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768059.633346:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768059.697281:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768059.697289:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768059.697291:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768059.697297:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768059.697302:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768059.697307:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768059.697310:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768059.697312:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768059.697343:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768060.593219:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768060.593225:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768060.593227:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768060.593232:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768060.593234:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768060.593235:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768060.593239:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768060.593239:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768060.593241:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768060.593242:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00100000:3.0:1775768060.593244:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768060.593246:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768060.593247:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768060.593250:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768060.593251:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768060.593253:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768060.593256:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768060.593256:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768060.593259:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768060.593260:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768060.593261:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768060.593264:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768060.593267:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768060.593269:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768060.593271:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768060.593272:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768060.593274:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768060.657246:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768060.657250:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768060.657254:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768060.657256:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768060.657264:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768060.657265:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768060.657267:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768060.657268:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768060.657270:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768060.657271:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768060.657297:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768060.657300:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768060.657302:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768060.721249:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768060.721260:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768060.721261:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768060.721265:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768060.721268:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768060.721271:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768060.721273:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768060.721274:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768060.721276:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768060.913251:0:14953:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4299985726, at 4299985728 00000100:00000040:1.0:1775768060.913262:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299985726, at 4299985728 00000100:00000010:0.0:1775768060.913272:0:14953:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cba540. 00000100:00000010:1.0:1775768060.913277:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cbab40. 00000100:00000001:3.0:1775768061.617245:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768061.617247:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768061.617250:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768061.617252:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768061.617252:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768061.617253:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768061.617256:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768061.617256:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:3.0:1775768061.617258:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768061.617259:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768061.617261:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:1.0:1775768061.617264:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768061.617265:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768061.617266:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768061.617269:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768061.617270:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768061.617271:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768061.617272:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768061.617273:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768061.617277:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768061.617279:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768061.617279:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768061.617282:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768061.617285:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768061.617286:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768061.617288:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768061.617290:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768061.681266:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768061.681271:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768061.681273:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768061.681275:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768061.681279:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768061.681280:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768061.681282:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768061.681283:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768061.681286:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768061.681287:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768061.681290:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768061.681291:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768061.681292:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768061.681293:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768061.681295:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768061.681295:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768061.681319:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768061.681321:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768061.681323:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768061.745234:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768061.745239:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768061.745240:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768061.745243:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768061.745245:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768061.745249:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768061.745251:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768061.745252:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768061.745254:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768062.641209:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768062.641214:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768062.641216:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768062.641221:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:3.0:1775768062.641224:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768062.641226:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768062.641228:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768062.641229:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768062.641231:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768062.641233:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768062.641234:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768062.641238:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768062.641238:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768062.641241:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768062.641242:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768062.641246:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768062.641247:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768062.641250:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768062.641260:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768062.641264:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768062.641265:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768062.641268:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768062.641270:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768062.641272:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768062.641274:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768062.641275:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768062.641277:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768062.705264:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768062.705270:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768062.705275:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768062.705276:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768062.705280:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768062.705281:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768062.705284:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768062.705285:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768062.705303:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768062.705305:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768062.705359:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768062.705362:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768062.705365:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768062.769248:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768062.769253:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768062.769255:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768062.769259:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768062.769261:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768062.769264:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768062.769266:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768062.769268:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768062.769270:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768063.665234:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768063.665235:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768063.665240:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768063.665240:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768063.665242:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768063.665242:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768063.665248:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:2.0:1775768063.665248:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768063.665252:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768063.665252:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768063.665257:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768063.665257:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768063.665259:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768063.665262:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768063.665265:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768063.665265:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768063.665266:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768063.665268:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768063.665269:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768063.665271:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768063.665273:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768063.665286:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775768063.665290:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768063.665292:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768063.665295:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768063.665297:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768063.665300:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768063.729250:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768063.729255:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768063.729260:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768063.729261:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768063.729264:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768063.729266:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768063.729268:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768063.729270:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768063.729272:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768063.729274:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768063.729303:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768063.729305:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768063.729307:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768063.793222:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768063.793227:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768063.793229:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768063.793233:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768063.793236:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768063.793238:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768063.793240:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768063.793242:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768063.793244:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768064.689303:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768064.689311:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768064.689311:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768064.689311:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768064.689317:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768064.689322:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:0.0:1775768064.689323:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768064.689323:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768064.689329:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768064.689330:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768064.689330:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768064.689336:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768064.689345:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:0.0:1775768064.689346:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00100000:2.0:1775768064.689346:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768064.689351:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768064.689357:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768064.689357:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768064.689357:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768064.689367:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768064.689368:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768064.689375:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768064.689376:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768064.689381:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768064.689382:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768064.689386:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768064.689392:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768064.689407:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768064.689411:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775768064.689413:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768064.689416:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768064.689419:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768064.689426:0:13471:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5317 00000400:00000200:0.0:1775768064.689434:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768064.689439:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768064.689441:0:13471:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768064.689444:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768064.689447:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768064.689450:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768064.689451:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768064.689454:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768064.689456:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:0.0:1775768064.689458:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768064.689462:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:0.0:1775768064.689464:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:0.0:1775768064.689465:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:0.0:1775768064.689468:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768064.689470:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768064.689472:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768064.689474:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768064.689476:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:0.0:1775768064.689478:0:13471:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768064.689480:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768064.689482:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768064.689484:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768064.689487:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768064.689489:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775768064.689490:0:13471:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768064.689497:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fc380. 00000020:00000040:0.0:1775768064.689500:0:13471:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:0.0:1775768064.689503:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768064.689504:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768064.689506:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768064.689508:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768064.689512:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768064.689517:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002a800. 00000100:00000001:0.0:1775768064.689522:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768064.689528:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fc380 x1862022294269440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768064.689534:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768064.689538:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768064.689541:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002957419c] to pc [ptlrpcd_rcv+-1] req@ffff9942099fc380 x1862022294269440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768064.689554:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768064.689557:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768064.689560:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775768064.689561:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768064.689563:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768064.689567:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768064.689570:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5317 00000400:00000200:0.0:1775768064.689575:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768064.689579:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:1.0:1775768064.689579:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:0.0:1775768064.689584:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1775768064.689585:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768064.689589:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768064.689602:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000040:1.0:1775768064.689602:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fc380 x1862022294269440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768064.689615:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775768064.689622:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000040:0.0:1775768064.689626:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768064.689626:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768064.689634:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 02000000:00000001:1.0:1775768064.689634:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768064.689637:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1775768064.689641:0:13471:0:(genops.c:811:class_conn2export()) Process entered 02000000:00000001:1.0:1775768064.689642:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1775768064.689646:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000100:00100000:1.0:1775768064.689649:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294269440:192.168.202.156@tcp:250:kworker.0 00000020:00000001:0.0:1775768064.689651:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:1.0:1775768064.689653:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000040:0.0:1775768064.689654:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:0.0:1775768064.689658:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 02000000:00000001:1.0:1775768064.689658:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768064.689662:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768064.689663:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 02000000:00000001:1.0:1775768064.689666:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:0.0:1775768064.689673:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 02000000:00000010:1.0:1775768064.689676:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543c800. 00000100:00000001:0.0:1775768064.689677:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1775768064.689681:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768064.689685:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768064.689688:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768064.689690:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000400:00000200:1.0:1775768064.689691:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00080000:0.0:1775768064.689699:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000400:00000010:1.0:1775768064.689699:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000100:00000001:0.0:1775768064.689705:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768064.689707:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:1.0:1775768064.689708:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 02000000:00000001:0.0:1775768064.689715:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000200:1.0:1775768064.689716:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294269440, portal 25 02000000:00000001:0.0:1775768064.689721:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768064.689722:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768064.689725:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641197440 : -112966068354176 : ffff9942099fc380) 00000100:00000001:0.0:1775768064.689727:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775768064.689733:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00000040:1.0:1775768064.689734:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fc380 x1862022294269440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768089 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:0.0:1775768064.689745:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768064.689756:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768064.689757:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768064.689762:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000100:00000010:0.0:1775768064.689764:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fca80. 00000400:00000010:1.0:1775768064.689764:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768064.689771:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294269440, offset 0 00000020:00000040:0.0:1775768064.689772:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:0.0:1775768064.689778:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000200:1.0:1775768064.689781:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 02000000:00000001:0.0:1775768064.689782:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768064.689786:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768064.689788:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768064.689790:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768064.689791:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000010:0.0:1775768064.689798:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002b800. 00000400:00000200:1.0:1775768064.689800:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00000001:0.0:1775768064.689803:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768064.689807:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00100000:0.0:1775768064.689812:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fca80 x1862022294269568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775768064.689813:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768064.689823:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768064.689831:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00000001:0.0:1775768064.689838:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1775768064.689842:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225278:225278:256:4294967295] 192.168.202.156@tcp LPNI seq info [225274:225274:8:4294967295] 00000100:00000001:0.0:1775768064.689846:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768064.689851:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000040:0.0:1775768064.689860:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000046a3ca2f] to pc [ptlrpcd_rcv+-1] req@ffff9942099fca80 x1862022294269568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775768064.689867:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768064.689877:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768064.689884:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000100:00000001:0.0:1775768064.689895:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775768064.689895:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768064.689905:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000100:00080000:0.0:1775768064.689906:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768064.689917:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775768064.689920:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768064.689925:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:1.1:1775768064.689925:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:0.0:1775768064.689928:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768064.689930:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5317 00000400:00000200:0.0:1775768064.689935:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768064.689940:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768064.689949:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000800:00000200:1.1:1775768064.689950:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000100:00000040:0.0:1775768064.689959:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000800:00000100:1.0:1775768064.689966:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000100:00000001:0.0:1775768064.689967:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000010:1.0:1775768064.689971:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000100:00000040:0.0:1775768064.689974:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768064.689977:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000200:1.0:1775768064.689979:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000001:0.0:1775768064.689980:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768064.689984:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:0.0:1775768064.689990:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:1.0:1775768064.689992:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000020:00000040:0.0:1775768064.689994:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000400:00000200:1.0:1775768064.689997:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000020:00000001:0.0:1775768064.689999:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:0.0:1775768064.690005:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000400:00000010:1.0:1775768064.690005:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000040:0.0:1775768064.690009:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768064.690013:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768064.690015:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768064.690015:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1775768064.690021:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768064.690023:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:0.0:1775768064.690027:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000200:1.0:1775768064.690028:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fc380 x1862022294269440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768089 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768064.690032:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768064.690039:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768064.690043:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768064.690049:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768064.690057:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775768064.690060:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775768064.690067:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768064.690074:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768064.690084:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fe680. 00000020:00000040:0.0:1775768064.690086:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000100:00000001:1.0:1775768064.690087:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1775768064.690089:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768064.690091:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768064.690093:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768064.690097:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:1.0:1775768064.690097:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fc380 x1862022294269440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768089 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1775768064.690100:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768064.690105:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220028400. 00000100:00000001:0.0:1775768064.690114:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768064.690122:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768064.690123:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fe680 x1862022294269696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775768064.690126:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768064.690133:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768064.690135:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768064.690137:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768064.690141:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ba41ba08] to pc [ptlrpcd_rcv+-1] req@ffff9942099fe680 x1862022294269696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775768064.690151:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768064.690191:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768064.690193:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299990846 00000100:00080000:0.0:1775768064.690195:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768064.690200:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775768064.690201:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768064.690202:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768064.690204:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000010:1.0:1775768064.690204:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cbab40. 00000100:00080000:0.0:1775768064.690208:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5317 00000400:00000200:0.0:1775768064.690212:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768064.690214:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768064.690215:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1775768064.690216:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000040:0.0:1775768064.690221:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768064.690223:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768064.690226:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768064.690229:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:1.0:1775768064.690229:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299990846 00000020:00000001:0.0:1775768064.690238:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768064.690240:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000100:00000010:1.0:1775768064.690240:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba8a0. 00000020:00000001:0.0:1775768064.690244:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:1.0:1775768064.690246:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000020:00000040:0.0:1775768064.690247:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000100:00000001:1.0:1775768064.690253:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1775768064.690254:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:1.0:1775768064.690259:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768064.690260:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:1.0:1775768064.690266:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000040:0.0:1775768064.690281:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00100000:1.0:1775768064.690286:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768064/real 1775768064] req@ffff9942099fc380 x1862022294269440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768064.690287:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768064.690300:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000001:1.0:1775768064.690303:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1775768064.690304:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000001:1.0:1775768064.690305:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768064.690308:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000020:00000040:0.0:1775768064.690311:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000400:00000200:1.0:1775768064.690313:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768064.690316:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00080000:0.0:1775768064.690320:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775768064.690321:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768064.690326:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fc380 x1862022294269440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768064.690327:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768064.690343:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000200:1.0:1775768064.690347:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fc380 x1862022294269440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1775768064.690349:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768064.690362:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768064.690362:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768064.690366:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768064.690367:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775768064.690370:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000008:00100000:0.0:1775768064.690371:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000100:00000001:1.0:1775768064.690372:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1775768064.690379:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768064.690379:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fc380 x1862022294269440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768064.690390:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768064.690398:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fdc00. 00000020:00000040:0.0:1775768064.690402:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00000100:00000001:1.0:1775768064.690402:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1775768064.690411:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768064.690412:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768064.690414:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768064.690418:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:1.0:1775768064.690419:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fc380 x1862022294269440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768089 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1775768064.690420:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768064.690425:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220029800. 00000100:00000001:0.0:1775768064.690430:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768064.690443:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fdc00 x1862022294269824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775768064.690445:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768064.690456:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768064.690460:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775768064.690462:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768064.690463:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768064.690465:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768064.690471:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001372d9ae] to pc [ptlrpcd_rcv+-1] req@ffff9942099fdc00 x1862022294269824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768064.690472:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768064.690477:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768064.690489:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:0.0:1775768064.690499:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768064.690504:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00080000:0.0:1775768064.690505:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775768064.690507:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00080000:0.0:1775768064.690514:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775768064.690521:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000040:1.0:1775768064.690521:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fc380 x1862022294269440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768089 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768064.690527:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768064.690536:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768064.690543:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5317 00000100:00100000:1.0:1775768064.690553:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294269440:192.168.202.156@tcp:250:kworker.0 00000400:00000200:0.0:1775768064.690558:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768064.690569:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768064.690571:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1775768064.690572:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768064.690577:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768064.690577:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1775768064.690585:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768064.690586:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fc380 x1862022294269440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768089 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775768064.690591:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768064.690601:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:1.0:1775768064.690602:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1775768064.690608:0:13471:0:(genops.c:811:class_conn2export()) Process entered 02000000:00000001:1.0:1775768064.690609:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000040:0.0:1775768064.690610:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 02000000:00000010:1.0:1775768064.690613:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543c800. 00000020:00000001:0.0:1775768064.690614:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768064.690617:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:0.0:1775768064.690619:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 02000000:00000001:1.0:1775768064.690620:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768064.690622:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:1.0:1775768064.690624:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000040:0.0:1775768064.690625:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768064.690628:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000040:1.0:1775768064.690628:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000100:00000040:0.0:1775768064.690635:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768064.690637:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775768064.690637:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 00000020:00000040:0.0:1775768064.690641:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 02000000:00000010:1.0:1775768064.690642:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002a800. 00000100:00080000:0.0:1775768064.690645:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768064.690647:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768064.690649:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775768064.690649:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768064.690654:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:0.0:1775768064.690655:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768064.690658:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775768064.690659:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fc380. 00000100:00000001:0.0:1775768064.690660:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775768064.690664:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000100:00000001:1.0:1775768064.690665:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000008:00000001:0.0:1775768064.690667:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768064.690668:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768064.690671:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768064.690673:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:0.0:1775768064.690675:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fce00. 00000020:00000040:0.0:1775768064.690678:0:13471:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:0.0:1775768064.690680:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768064.690681:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768064.690682:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768064.690682:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768064.690684:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768064.690685:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 02000000:00000010:0.0:1775768064.690687:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002a800. 00000100:00000001:1.0:1775768064.690688:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768064.690692:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768064.690693:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:0.0:1775768064.690698:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fce00 x1862022294269952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775768064.690698:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768064.690701:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768064.690713:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fca80 x1862022294269568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768064.690733:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768064.690739:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768064.690749:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000060e5e9d1] to pc [ptlrpcd_rcv+-1] req@ffff9942099fce00 x1862022294269952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768064.690762:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768064.690766:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768064.690778:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1775768064.690779:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768064.690783:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1775768064.690785:0:13471:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5327) 02000000:00000001:1.0:1775768064.690786:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768064.690797:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294269568:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768064.690800:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768064.690802:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768064.690804:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768064.690805:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768064.690808:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543c800. 02000000:00000001:1.0:1775768064.690809:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768064.690813:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768064.690815:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000400:00000010:1.0:1775768064.690818:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000200:1.0:1775768064.690820:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294269568, portal 10 00000100:00000001:1.0:1775768064.690821:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768064.690822:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641199232 : -112966068352384 : ffff9942099fca80) 00000100:00000040:1.0:1775768064.690827:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fca80 x1862022294269568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768089 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768064.690832:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768064.690834:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768064.690835:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768064.690837:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294269568, offset 0 00000400:00000200:1.0:1775768064.690839:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768064.690841:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768064.690844:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768064.690846:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768064.690847:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768064.690849:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768064.690850:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768064.690853:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225279:225279:256:4294967295] 192.168.202.156@tcp LPNI seq info [225275:225275:8:4294967295] 00000400:00000200:1.0:1775768064.690855:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768064.690859:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768064.690862:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768064.690864:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000800:00000200:1.0:1775768064.690867:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768064.690869:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775768064.690872:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768064.690874:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000100:1.0:1775768064.690876:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768064.690878:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768064.690880:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768064.690882:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768064.690884:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768064.690885:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768064.690887:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768064.690888:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fca80 x1862022294269568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768089 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768064.690894:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768064.690895:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fca80 x1862022294269568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768089 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768064.690904:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768064.690904:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768064.690906:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768064.690909:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:1.0:1775768064.690911:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768064.690913:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299989523 00000100:00000001:1.0:1775768064.690915:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299990846 00000100:00000001:1.0:1775768064.690916:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299989523 00000100:00000001:1.0:1775768064.690918:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299990846 00000100:00000001:1.0:1775768064.690919:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768064.690920:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768064.690921:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768064.690923:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768064/real 1775768064] req@ffff9942099fca80 x1862022294269568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768064.690927:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768064.690928:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768064.690929:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768064.690930:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768064.690931:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768064.690932:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768064.690934:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fca80 x1862022294269568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768064.690939:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fca80 x1862022294269568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768064.690942:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768064.690943:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768064.690943:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768064.690944:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768064.690946:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fca80 x1862022294269568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768064.690951:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768064.690953:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fca80 x1862022294269568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768089 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768064.690957:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768064.690958:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768064.690959:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768064.690960:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768064.690961:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768064.690963:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768064.690964:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768064.690966:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768064.690967:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768064.690969:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fca80 x1862022294269568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768089 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768064.690973:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294269568:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768064.690975:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768064.690977:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fe680 x1862022294269696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768064.690982:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768064.690982:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768064.690984:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768064.690985:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768064.690986:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768064.690987:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294269696:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768064.690989:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768064.690990:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768064.690991:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768064.690992:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768064.690994:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543e800. 02000000:00000001:1.0:1775768064.690995:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768064.690997:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768064.690998:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000400:00000010:1.0:1775768064.691000:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000200:1.0:1775768064.691002:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294269696, portal 10 00000100:00000001:1.0:1775768064.691003:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768064.691004:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641206400 : -112966068345216 : ffff9942099fe680) 00000100:00000040:1.0:1775768064.691008:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fe680 x1862022294269696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768089 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768064.691011:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768064.691013:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768064.691014:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768064.691015:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294269696, offset 0 00000400:00000200:1.0:1775768064.691017:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768064.691019:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768064.691021:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768064.691022:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768064.691023:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768064.691024:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768064.691026:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768064.691027:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225280:225280:256:4294967295] 192.168.202.156@tcp LPNI seq info [225276:225276:8:4294967295] 00000400:00000200:1.0:1775768064.691030:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768064.691033:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768064.691036:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768064.691037:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000800:00000200:1.0:1775768064.691039:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768064.691041:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775768064.691044:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768064.691045:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000100:1.0:1775768064.691048:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768064.691049:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768064.691051:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768064.691053:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768064.691054:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768064.691055:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768064.691056:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768064.691058:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fe680 x1862022294269696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768089 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768064.691063:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768064.691064:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fe680 x1862022294269696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768089 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768064.691068:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768064.691069:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768064.691070:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768064.691072:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:1.0:1775768064.691074:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768064.691075:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299989515 00000100:00000001:1.0:1775768064.691076:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299990846 00000100:00000001:1.0:1775768064.691078:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299989515 00000100:00000001:1.0:1775768064.691079:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299990846 00000100:00000001:1.0:1775768064.691080:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768064.691081:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768064.691082:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768064.691084:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768064/real 1775768064] req@ffff9942099fe680 x1862022294269696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768064.691088:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768064.691088:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768064.691089:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768064.691090:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768064.691091:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768064.691092:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768064.691094:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fe680 x1862022294269696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768064.691099:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fe680 x1862022294269696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768064.691102:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768064.691103:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768064.691104:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768064.691105:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768064.691106:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fe680 x1862022294269696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768064.691110:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768064.691111:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fe680 x1862022294269696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768089 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768064.691115:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768064.691116:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768064.691117:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768064.691117:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768064.691118:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768064.691120:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768064.691121:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768064.691123:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768064.691124:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768064.691126:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fe680 x1862022294269696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768089 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768064.691130:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294269696:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768064.691132:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768064.691135:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fdc00 x1862022294269824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768064.691139:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768064.691140:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768064.691141:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768064.691142:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768064.691143:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768064.691145:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294269824:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768064.691146:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768064.691147:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768064.691148:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768064.691149:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768064.691151:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543e400. 02000000:00000001:1.0:1775768064.691151:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768064.691154:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768064.691172:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000400:00000010:1.0:1775768064.691175:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000200:1.0:1775768064.691176:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294269824, portal 4 00000100:00000001:1.0:1775768064.691177:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768064.691178:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641203712 : -112966068347904 : ffff9942099fdc00) 00000100:00000040:1.0:1775768064.691181:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fdc00 x1862022294269824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768089 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768064.691186:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768064.691187:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768064.691188:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768064.691189:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294269824, offset 0 00000400:00000200:1.0:1775768064.691191:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768064.691193:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768064.691195:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768064.691196:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768064.691198:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768064.691199:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768064.691200:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768064.691203:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225281:225281:256:4294967295] 192.168.202.156@tcp LPNI seq info [225277:225277:8:4294967295] 00000400:00000200:1.0:1775768064.691205:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768064.691209:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768064.691212:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768064.691214:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000800:00000200:1.0:1775768064.691216:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768064.691218:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775768064.691220:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768064.691222:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000100:1.0:1775768064.691224:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768064.691225:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768064.691227:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768064.691229:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768064.691230:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768064.691231:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768064.691233:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768064.691235:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fdc00 x1862022294269824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768089 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768064.691239:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768064.691241:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fdc00 x1862022294269824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768089 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768064.691244:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768064.691245:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768064.691247:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768064.691248:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:1.0:1775768064.691250:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768064.691251:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299989539 00000100:00000001:1.0:1775768064.691252:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299990846 00000100:00000001:1.0:1775768064.691253:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299989539 00000100:00000001:1.0:1775768064.691255:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299990846 00000100:00000001:1.0:1775768064.691257:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768064.691258:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768064.691259:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768064.691261:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768064/real 1775768064] req@ffff9942099fdc00 x1862022294269824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768064.691266:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768064.691266:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768064.691279:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768064.691281:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768064.691282:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768064.691283:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768064.691285:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fdc00 x1862022294269824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768064.691290:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fdc00 x1862022294269824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768064.691294:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768064.691295:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768064.691296:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768064.691297:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768064.691298:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fdc00 x1862022294269824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768064.691302:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768064.691304:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fdc00 x1862022294269824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768089 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768064.691308:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768064.691310:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768064.691311:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768064.691311:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768064.691312:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768064.691315:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768064.691316:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768064.691319:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768064.691320:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768064.691322:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fdc00 x1862022294269824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768089 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768064.691326:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294269824:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768064.691328:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768064.691329:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768064.691331:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fca80 x1862022294269568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768089 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768064.691334:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768064.691335:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768064.691336:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543c800. 02000000:00000001:1.0:1775768064.691338:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768064.691338:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768064.691339:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:1.0:1775768064.691341:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768064.691342:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002b800. 02000000:00000001:1.0:1775768064.691344:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768064.691345:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768064.691346:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fca80. 00000100:00000001:1.0:1775768064.691347:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768064.691348:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768064.691349:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768064.691351:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fe680 x1862022294269696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768089 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768064.691354:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768064.691355:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768064.691356:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543e800. 02000000:00000001:1.0:1775768064.691357:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768064.691358:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768064.691358:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:1.0:1775768064.691359:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768064.691360:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028400. 02000000:00000001:1.0:1775768064.691362:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768064.691363:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768064.691364:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fe680. 00000100:00000001:1.0:1775768064.691365:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768064.691366:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768064.691367:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768064.691368:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fdc00 x1862022294269824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768089 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768064.691371:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768064.691372:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768064.691373:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543e400. 02000000:00000001:1.0:1775768064.691373:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768064.691374:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768064.691375:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:1.0:1775768064.691375:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768064.691376:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029800. 02000000:00000001:1.0:1775768064.691377:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768064.691378:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768064.691378:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fdc00. 00000100:00000001:1.0:1775768064.691380:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768064.691380:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768064.691381:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768064.691383:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768064.691384:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768064.691385:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768064.691386:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768064.691386:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768064.691388:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fce00 x1862022294269952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768064.691392:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768064.691393:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768064.691394:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768064.691395:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768064.691396:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768064.691398:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294269952:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768064.691400:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768064.691401:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768064.691402:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768064.691403:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768064.691405:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543e400. 02000000:00000001:1.0:1775768064.691406:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768064.691408:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768064.691410:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000400:00000010:1.0:1775768064.691412:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000200:1.0:1775768064.691414:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294269952, portal 4 00000100:00000001:1.0:1775768064.691415:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768064.691416:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641200128 : -112966068351488 : ffff9942099fce00) 00000100:00000040:1.0:1775768064.691420:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fce00 x1862022294269952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768089 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768064.691424:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768064.691425:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768064.691426:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768064.691428:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294269952, offset 0 00000400:00000200:1.0:1775768064.691430:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768064.691432:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768064.691434:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768064.691435:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768064.691436:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768064.691438:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768064.691439:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768064.691441:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225282:225282:256:4294967295] 192.168.202.156@tcp LPNI seq info [225278:225278:8:4294967295] 00000400:00000200:1.0:1775768064.691444:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768064.691447:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768064.691449:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768064.691451:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000800:00000200:1.0:1775768064.691453:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768064.691455:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775768064.691458:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768064.691459:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000100:1.0:1775768064.691462:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768064.691463:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768064.691464:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768064.691467:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768064.691468:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768064.691469:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768064.691471:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768064.691472:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fce00 x1862022294269952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768089 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768064.691477:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768064.691479:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fce00 x1862022294269952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768089 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768064.691482:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768064.691483:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768064.691484:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768064.691486:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:1.0:1775768064.691487:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768064.691488:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299989542 00000100:00000001:1.0:1775768064.691490:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299990846 00000100:00000001:1.0:1775768064.691491:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299989542 00000100:00000001:1.0:1775768064.691493:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299990846 00000100:00000001:1.0:1775768064.691494:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768064.691495:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768064.691496:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768064.691498:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768064/real 1775768064] req@ffff9942099fce00 x1862022294269952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768064.691502:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768064.691502:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768064.691504:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768064.691505:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768064.691506:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768064.691507:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768064.691509:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fce00 x1862022294269952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768064.691513:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fce00 x1862022294269952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768064.691516:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768064.691517:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768064.691518:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768064.691519:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768064.691520:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fce00 x1862022294269952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768064.691523:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768064.691525:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fce00 x1862022294269952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768089 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768064.691529:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768064.691530:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768064.691531:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768064.691531:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768064.691532:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768064.691535:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768064.691537:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768064.691539:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768064.691540:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768064.691542:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fce00 x1862022294269952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768089 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768064.691546:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294269952:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768064.691549:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768064.691550:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768064.691552:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fce00 x1862022294269952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768089 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768064.691555:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768064.691556:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768064.691557:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543e400. 02000000:00000001:1.0:1775768064.691558:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768064.691558:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768064.691559:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:1.0:1775768064.691560:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768064.691561:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002a800. 02000000:00000001:1.0:1775768064.691562:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768064.691563:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768064.691564:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fce00. 00000100:00000001:1.0:1775768064.691565:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768064.691566:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768064.691567:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768064.691568:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768064.691569:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768064.691570:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768064.691571:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768064.753299:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768064.753304:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768064.753308:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768064.753309:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768064.753312:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768064.753313:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768064.753315:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768064.753316:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768064.753318:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768064.753319:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768064.753374:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768064.753377:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768064.753379:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768064.817257:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768064.817261:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768064.817262:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768064.817266:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768064.817281:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768064.817285:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768064.817286:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768064.817288:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768064.817290:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768065.713324:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768065.713333:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768065.713335:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768065.713341:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775768065.713346:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768065.713351:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768065.713354:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768065.713358:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768065.713361:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768065.713588:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768065.713595:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768065.713597:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768065.713602:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768065.713606:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768065.713609:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768065.713612:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768065.713614:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768065.713616:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768065.715635:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768065.715640:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768065.715642:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768065.715647:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768065.715650:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768065.715653:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768065.715655:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768065.715657:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768065.715659:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768065.777307:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768065.777312:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768065.777317:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768065.777320:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768065.777323:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768065.777327:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768065.777331:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768065.777333:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768065.777335:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768065.777339:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768065.777343:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768065.777344:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768065.777345:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768065.777347:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768065.777348:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768065.777349:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768065.777352:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768065.777353:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768065.777356:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768065.777357:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768065.777358:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768065.777359:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768065.777361:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768065.777362:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768065.777364:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768065.777366:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768065.777369:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768065.777370:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768065.777371:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768065.777372:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768065.777374:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768065.777375:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768065.777377:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768065.777379:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768065.777402:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768065.777404:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768065.777407:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768065.841292:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768065.841297:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768065.841299:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768065.841302:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768065.841305:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768065.841308:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768065.841310:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768065.841312:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768065.841314:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768066.033290:0:14953:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4299990846, at 4299990848 00000100:00000040:1.0:1775768066.033291:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299990846, at 4299990848 00000100:00000010:1.0:1775768066.033309:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cba8a0. 00000100:00000010:0.0:1775768066.033316:0:14953:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cbab40. 00000100:00000001:3.0:1775768066.737212:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768066.737220:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768066.737222:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768066.737227:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768066.737230:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768066.737235:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768066.737237:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768066.737240:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768066.737242:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768066.737260:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768066.737265:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768066.737267:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768066.737270:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768066.737273:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768066.737276:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768066.737276:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768066.737278:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768066.737281:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768066.737283:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768066.737284:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768066.737287:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768066.737292:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768066.737295:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768066.737299:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768066.737301:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768066.737303:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768066.737306:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768066.801288:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768066.801293:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768066.801299:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768066.801301:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768066.801304:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768066.801305:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768066.801308:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768066.801308:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768066.801311:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768066.801312:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768066.801384:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768066.801389:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768066.801394:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768066.865271:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768066.865278:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768066.865280:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768066.865285:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768066.865288:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768066.865293:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768066.865295:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768066.865297:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768066.865299:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768067.761212:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768067.761220:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768067.761222:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768067.761227:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:3.0:1775768067.761228:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768067.761234:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768067.761234:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768067.761251:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768067.761253:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768067.761257:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:1.0:1775768067.761257:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768067.761258:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768067.761261:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768067.761264:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768067.761265:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768067.761266:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768067.761269:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768067.761270:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768067.761276:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00100000:0.0:1775768067.761277:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768067.761279:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768067.761281:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768067.761283:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768067.761284:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768067.761288:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768067.761290:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768067.761292:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768067.825302:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768067.825308:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768067.825314:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768067.825316:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768067.825319:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768067.825320:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768067.825323:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768067.825324:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768067.825326:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768067.825327:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768067.825347:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768067.825349:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768067.825352:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768067.889331:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768067.889339:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768067.889341:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768067.889347:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768067.889350:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768067.889355:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768067.889357:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768067.889360:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768067.889363:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768068.785217:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768068.785225:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768068.785240:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768068.785246:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768068.785247:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768068.785251:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768068.785252:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768068.785256:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768068.785259:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768068.785259:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768068.785264:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768068.785265:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768068.785267:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768068.785269:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768068.785274:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768068.785275:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768068.785278:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768068.785281:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768068.785282:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768068.785289:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768068.785292:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768068.785297:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768068.785302:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768068.785305:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768068.785307:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768068.785309:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768068.785311:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768068.849306:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768068.849315:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768068.849325:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768068.849326:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768068.849329:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768068.849330:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768068.849333:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768068.849334:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768068.849336:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768068.849337:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768068.849393:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768068.849397:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768068.849401:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768068.913273:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768068.913281:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768068.913283:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768068.913288:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768068.913292:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768068.913296:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768068.913298:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768068.913301:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768068.913303:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768069.809246:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768069.809253:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768069.809256:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768069.809263:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768069.809269:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768069.809273:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768069.809276:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768069.809279:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768069.809282:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768069.809312:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768069.809319:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768069.809321:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768069.809324:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768069.809327:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768069.809330:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768069.809331:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768069.809333:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768069.809335:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768069.809381:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768069.809389:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775768069.809393:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768069.809399:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768069.809404:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768069.809414:0:13471:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5322 00000400:00000200:0.0:1775768069.809435:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768069.809445:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768069.809449:0:13471:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768069.809453:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768069.809457:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768069.809460:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768069.809463:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768069.809468:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768069.809470:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:0.0:1775768069.809473:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768069.809476:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:0.0:1775768069.809479:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:0.0:1775768069.809481:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:0.0:1775768069.809485:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768069.809487:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768069.809489:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768069.809491:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768069.809494:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:0.0:1775768069.809498:0:13471:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768069.809502:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768069.809506:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768069.809510:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768069.809514:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768069.809517:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775768069.809520:0:13471:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768069.809530:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fea00. 00000020:00000040:0.0:1775768069.809534:0:13471:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:0.0:1775768069.809539:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768069.809541:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768069.809545:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768069.809548:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768069.809554:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768069.809564:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220028000. 00000100:00000001:0.0:1775768069.809573:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768069.809585:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fea00 x1862022294270080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768069.809599:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768069.809606:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768069.809612:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008d4eaacc] to pc [ptlrpcd_rcv+-1] req@ffff9942099fea00 x1862022294270080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768069.809637:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768069.809641:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768069.809645:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775768069.809647:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768069.809649:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768069.809653:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768069.809656:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5322 00000100:00000001:1.0:1775768069.809656:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768069.809661:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768069.809663:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000400:00000200:0.0:1775768069.809665:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768069.809672:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000040:1.0:1775768069.809678:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fea00 x1862022294270080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768069.809680:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768069.809697:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768069.809706:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768069.809707:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775768069.809708:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768069.809713:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:0.0:1775768069.809714:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000001:1.0:1775768069.809717:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768069.809723:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 02000000:00000001:1.0:1775768069.809723:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768069.809734:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768069.809739:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000100:00100000:1.0:1775768069.809739:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294270080:192.168.202.156@tcp:250:kworker.0 00000020:00000001:0.0:1775768069.809748:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768069.809755:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000100:00000001:1.0:1775768069.809757:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000001:0.0:1775768069.809761:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 02000000:00000001:1.0:1775768069.809762:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:0.0:1775768069.809768:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 02000000:00000001:1.0:1775768069.809768:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768069.809775:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:0.0:1775768069.809780:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 02000000:00000010:1.0:1775768069.809781:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543e800. 00000100:00000001:0.0:1775768069.809790:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1775768069.809790:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768069.809801:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000200:1.0:1775768069.809808:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768069.809809:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768069.809816:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000400:00000010:1.0:1775768069.809817:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000100:00080000:0.0:1775768069.809824:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768069.809827:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768069.809829:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768069.809837:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:1.0:1775768069.809838:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 02000000:00000001:0.0:1775768069.809842:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768069.809846:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000200:1.0:1775768069.809850:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294270080, portal 25 00000008:00100000:0.0:1775768069.809853:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1775768069.809855:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768069.809857:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641207296 : -112966068344320 : ffff9942099fea00) 00000008:00000001:0.0:1775768069.809861:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768069.809866:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768069.809870:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fea00 x1862022294270080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768094 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1775768069.809875:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fce00. 00000020:00000040:0.0:1775768069.809881:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:0.0:1775768069.809888:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768069.809890:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768069.809892:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768069.809893:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1775768069.809896:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:0.0:1775768069.809897:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768069.809904:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220029400. 00000100:00000040:1.0:1775768069.809907:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000100:00000001:0.0:1775768069.809913:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775768069.809914:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768069.809923:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294270080, offset 0 00000100:00100000:0.0:1775768069.809927:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fce00 x1862022294270208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775768069.809938:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00000001:0.0:1775768069.809948:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1775768069.809950:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768069.809956:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768069.809962:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000060e5e9d1] to pc [ptlrpcd_rcv+-1] req@ffff9942099fce00 x1862022294270208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775768069.809964:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00000001:0.0:1775768069.809984:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768069.809985:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:0.0:1775768069.809988:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768069.809996:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000200:1.0:1775768069.809996:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000100:00000001:0.0:1775768069.810002:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768069.810004:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1775768069.810010:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1775768069.810012:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768069.810018:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5322 00000400:00000200:1.0:1775768069.810018:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768069.810031:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225283:225283:256:4294967295] 192.168.202.156@tcp LPNI seq info [225279:225279:8:4294967295] 00000400:00000200:1.0:1775768069.810036:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768069.810045:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768069.810050:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768069.810052:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768069.810062:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000800:00000010:1.0:1775768069.810062:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000100:00080000:0.0:1775768069.810067:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768069.810072:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000800:00000200:1.0:1775768069.810077:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:0.0:1775768069.810079:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768069.810088:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000800:00000200:1.0:1775768069.810088:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000100:00000001:0.0:1775768069.810091:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768069.810097:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768069.810102:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:0.0:1775768069.810108:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000800:00000200:1.1:1775768069.810109:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000040:0.0:1775768069.810112:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000800:00000200:1.1:1775768069.810115:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000020:00000001:0.0:1775768069.810117:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:0.0:1775768069.810123:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000800:00000100:1.0:1775768069.810128:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000100:00000040:0.0:1775768069.810134:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000800:00000010:1.0:1775768069.810135:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000100:00000001:0.0:1775768069.810143:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1775768069.810147:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:0.0:1775768069.810148:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768069.810185:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768069.810191:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000400:00000200:1.0:1775768069.810193:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000100:00080000:0.0:1775768069.810202:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000400:00000200:1.0:1775768069.810203:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000100:00000001:0.0:1775768069.810212:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775768069.810212:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 02000000:00000001:0.0:1775768069.810232:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1775768069.810236:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 02000000:00000001:0.0:1775768069.810239:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768069.810240:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1775768069.810240:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768069.810246:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768069.810247:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768069.810260:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000200:1.0:1775768069.810261:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fea00 x1862022294270080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768094 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768069.810262:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000008:00100000:0.0:1775768069.810266:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1775768069.810268:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000008:00000001:0.0:1775768069.810273:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768069.810273:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768069.810284:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768069.810285:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768069.810292:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:0.0:1775768069.810294:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fdc00. 00000100:00000001:2.0:1775768069.810298:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1775768069.810305:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:0.0:1775768069.810310:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1775768069.810311:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1775768069.810312:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768069.810316:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768069.810318:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768069.810321:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768069.810326:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fea00 x1862022294270080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768094 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1775768069.810328:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002bc00. 00000100:00000001:0.0:1775768069.810334:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768069.810344:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fdc00 x1862022294270336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775768069.810357:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768069.810366:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768069.810367:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768069.810370:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768069.810375:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768069.810378:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001372d9ae] to pc [ptlrpcd_rcv+-1] req@ffff9942099fdc00 x1862022294270336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775768069.810389:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768069.810398:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4299995966 00000100:00000001:0.0:1775768069.810403:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768069.810407:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000010:1.0:1775768069.810407:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba8a0. 00000100:00080000:0.0:1775768069.810414:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775768069.810415:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768069.810417:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768069.810417:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775768069.810427:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775768069.810430:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4299995966 00000100:00080000:0.0:1775768069.810431:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5322 00000100:00000010:1.0:1775768069.810438:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba840. 00000400:00000200:0.0:1775768069.810439:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768069.810445:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768069.810447:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1775768069.810448:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000040:0.0:1775768069.810453:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768069.810457:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775768069.810457:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1775768069.810461:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768069.810467:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:1.0:1775768069.810468:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768069.810470:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768069.810475:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000100:00000001:1.0:1775768069.810477:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000020:00000001:0.0:1775768069.810478:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768069.810481:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:0.0:1775768069.810483:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:0.0:1775768069.810488:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00100000:1.0:1775768069.810488:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768069/real 1775768069] req@ffff9942099fea00 x1862022294270080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775768069.810501:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768069.810513:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768069.810516:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000001:1.0:1775768069.810517:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768069.810519:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1775768069.810521:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768069.810525:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000400:00000010:1.0:1775768069.810525:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000100:00080000:0.0:1775768069.810531:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768069.810534:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768069.810535:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 02000000:00000001:0.0:1775768069.810536:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768069.810542:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:1.0:1775768069.810543:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 02000000:00000001:0.0:1775768069.810546:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768069.810549:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775768069.810552:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000100:00000001:1.0:1775768069.810553:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000008:00000001:0.0:1775768069.810554:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768069.810557:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768069.810560:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fe680. 00000020:00000040:0.0:1775768069.810563:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00000100:00000200:1.0:1775768069.810563:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fea00 x1862022294270080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1775768069.810569:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768069.810571:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768069.810577:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768069.810579:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768069.810582:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768069.810590:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002a800. 00000100:00000001:0.0:1775768069.810597:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768069.810604:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fe680 x1862022294270464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:1.0:1775768069.810608:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fea00 x1862022294270080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768069.810626:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768069.810631:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768069.810641:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ba41ba08] to pc [ptlrpcd_rcv+-1] req@ffff9942099fe680 x1862022294270464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768069.810651:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768069.810657:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768069.810663:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768069.810664:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768069.810667:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1775768069.810671:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775768069.810678:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fea00 x1862022294270080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768069.810682:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775768069.810689:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768069.810700:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768069.810700:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768069.810709:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000040:1.0:1775768069.810709:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fea00 x1862022294270080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768094 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768069.810721:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5322 00000100:00000001:1.0:1775768069.810725:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000400:00000200:0.0:1775768069.810732:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768069.810733:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768069.810737:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768069.810738:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1775768069.810739:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768069.810742:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1775768069.810742:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768069.810746:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:0.0:1775768069.810750:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00080000:1.0:1775768069.810754:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:0.0:1775768069.810755:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768069.810762:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00080000:1.0:1775768069.810767:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768069.810768:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:1.0:1775768069.810772:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000001:0.0:1775768069.810773:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768069.810776:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000100:00000040:1.0:1775768069.810781:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fea00 x1862022294270080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768094 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775768069.810782:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768069.810788:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:0.0:1775768069.810799:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:0.0:1775768069.810804:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:0.0:1775768069.810816:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768069.810821:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:1.0:1775768069.810822:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294270080:192.168.202.156@tcp:250:kworker.0 00000100:00000040:0.0:1775768069.810830:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768069.810838:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768069.810848:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00000001:1.0:1775768069.810850:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768069.810855:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00080000:0.0:1775768069.810859:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000040:1.0:1775768069.810863:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fea00 x1862022294270080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768094 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768069.810869:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768069.810876:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768069.810883:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768069.810884:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768069.810890:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768069.810893:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:1.0:1775768069.810893:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000008:00100000:0.0:1775768069.810897:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 02000000:00000010:1.0:1775768069.810899:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543e800. 00000008:00000001:0.0:1775768069.810901:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768069.810906:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768069.810910:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000010:0.0:1775768069.810911:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fca80. 00000020:00000040:0.0:1775768069.810915:0:13471:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:1.0:1775768069.810915:0:2399:0:(genops.c:1236:class_import_put()) Process entered 02000000:00000001:0.0:1775768069.810919:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000020:00000040:1.0:1775768069.810920:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 02000000:00000001:0.0:1775768069.810922:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768069.810928:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000020:00000001:1.0:1775768069.810929:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000001:0.0:1775768069.810930:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768069.810935:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775768069.810935:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028000. 02000000:00000010:0.0:1775768069.810947:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220029800. 02000000:00000001:1.0:1775768069.810948:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:0.0:1775768069.810955:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768069.810955:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768069.810960:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fea00. 00000100:00100000:0.0:1775768069.810964:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fca80 x1862022294270592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775768069.810970:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768069.810975:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768069.810983:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768069.810986:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768069.810990:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768069.810995:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:0.0:1775768069.810996:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000046a3ca2f] to pc [ptlrpcd_rcv+-1] req@ffff9942099fca80 x1862022294270592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768069.810999:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768069.811004:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768069.811008:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768069.811010:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768069.811020:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fce00 x1862022294270208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768069.811027:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768069.811031:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768069.811034:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768069.811037:0:13471:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5332) 02000000:00000001:1.0:1775768069.811038:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768069.811042:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768069.811047:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768069.811051:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294270208:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768069.811055:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768069.811058:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768069.811061:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768069.811063:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768069.811068:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543e800. 02000000:00000001:1.0:1775768069.811071:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768069.811076:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768069.811080:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000400:00000010:1.0:1775768069.811087:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000200:1.0:1775768069.811091:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294270208, portal 10 00000100:00000001:1.0:1775768069.811093:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768069.811094:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641200128 : -112966068351488 : ffff9942099fce00) 00000100:00000040:1.0:1775768069.811103:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fce00 x1862022294270208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768094 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768069.811111:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768069.811115:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768069.811119:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768069.811122:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294270208, offset 0 00000400:00000200:1.0:1775768069.811132:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768069.811138:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768069.811144:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768069.811147:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768069.811150:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768069.811153:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768069.811176:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768069.811179:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225284:225284:256:4294967295] 192.168.202.156@tcp LPNI seq info [225280:225280:8:4294967295] 00000400:00000200:1.0:1775768069.811183:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768069.811188:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768069.811192:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768069.811195:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000800:00000200:1.0:1775768069.811198:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768069.811200:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775768069.811203:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768069.811205:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1775768069.811208:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768069.811209:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768069.811212:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768069.811215:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768069.811217:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768069.811230:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768069.811234:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768069.811238:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fce00 x1862022294270208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768094 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768069.811247:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768069.811251:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fce00 x1862022294270208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768094 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768069.811258:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768069.811259:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768069.811263:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768069.811267:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:1.0:1775768069.811271:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768069.811273:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299994644 00000100:00000001:1.0:1775768069.811276:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299995966 00000100:00000001:1.0:1775768069.811278:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299994644 00000100:00000001:1.0:1775768069.811281:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299995966 00000100:00000001:1.0:1775768069.811283:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768069.811285:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768069.811287:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768069.811291:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768069/real 1775768069] req@ffff9942099fce00 x1862022294270208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768069.811297:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768069.811298:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768069.811299:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768069.811301:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768069.811302:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768069.811304:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768069.811306:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fce00 x1862022294270208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768069.811316:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fce00 x1862022294270208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768069.811322:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768069.811324:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768069.811326:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768069.811328:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768069.811331:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fce00 x1862022294270208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768069.811339:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768069.811343:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fce00 x1862022294270208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768094 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768069.811350:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768069.811352:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768069.811354:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768069.811355:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768069.811357:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768069.811360:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768069.811362:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768069.811366:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768069.811368:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768069.811372:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fce00 x1862022294270208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768094 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768069.811381:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294270208:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768069.811385:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768069.811389:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fdc00 x1862022294270336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768069.811399:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768069.811401:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768069.811403:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768069.811404:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768069.811405:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768069.811407:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294270336:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768069.811409:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768069.811411:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768069.811413:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768069.811414:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768069.811417:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543c800. 02000000:00000001:1.0:1775768069.811418:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768069.811421:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768069.811422:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000400:00000010:1.0:1775768069.811426:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000200:1.0:1775768069.811429:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294270336, portal 10 00000100:00000001:1.0:1775768069.811431:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768069.811432:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641203712 : -112966068347904 : ffff9942099fdc00) 00000100:00000040:1.0:1775768069.811438:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fdc00 x1862022294270336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768094 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768069.811445:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768069.811447:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768069.811449:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768069.811452:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294270336, offset 0 00000400:00000200:1.0:1775768069.811456:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768069.811460:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768069.811464:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768069.811467:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768069.811469:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768069.811471:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768069.811474:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768069.811478:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225285:225285:256:4294967295] 192.168.202.156@tcp LPNI seq info [225281:225281:8:4294967295] 00000400:00000200:1.0:1775768069.811482:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768069.811487:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768069.811491:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768069.811493:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000800:00000200:1.0:1775768069.811496:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768069.811498:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775768069.811501:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768069.811503:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1775768069.811505:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768069.811506:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768069.811509:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768069.811511:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768069.811513:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768069.811514:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768069.811516:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768069.811518:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fdc00 x1862022294270336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768094 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768069.811524:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768069.811526:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fdc00 x1862022294270336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768094 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768069.811530:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768069.811531:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768069.811533:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768069.811536:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:1.0:1775768069.811538:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768069.811539:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299994636 00000100:00000001:1.0:1775768069.811541:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299995966 00000100:00000001:1.0:1775768069.811542:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299994636 00000100:00000001:1.0:1775768069.811544:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299995966 00000100:00000001:1.0:1775768069.811545:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768069.811547:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768069.811548:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768069.811550:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768069/real 1775768069] req@ffff9942099fdc00 x1862022294270336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768069.811556:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768069.811556:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768069.811557:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768069.811559:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768069.811560:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768069.811561:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768069.811563:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fdc00 x1862022294270336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768069.811569:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fdc00 x1862022294270336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768069.811573:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768069.811574:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768069.811575:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768069.811576:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768069.811578:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fdc00 x1862022294270336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768069.811582:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768069.811584:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fdc00 x1862022294270336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768094 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768069.811588:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768069.811590:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768069.811591:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768069.811592:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768069.811593:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768069.811595:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768069.811597:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768069.811599:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768069.811601:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768069.811603:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fdc00 x1862022294270336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768094 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768069.811608:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294270336:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768069.811610:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768069.811613:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fe680 x1862022294270464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768069.811617:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768069.811618:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768069.811619:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768069.811620:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768069.811622:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768069.811623:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294270464:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768069.811626:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768069.811627:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768069.811629:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768069.811630:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768069.811632:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543c400. 02000000:00000001:1.0:1775768069.811633:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768069.811635:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768069.811637:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000400:00000010:1.0:1775768069.811639:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000200:1.0:1775768069.811641:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294270464, portal 4 00000100:00000001:1.0:1775768069.811642:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768069.811643:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641206400 : -112966068345216 : ffff9942099fe680) 00000100:00000040:1.0:1775768069.811647:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fe680 x1862022294270464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768094 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768069.811651:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768069.811653:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768069.811654:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768069.811656:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294270464, offset 0 00000400:00000200:1.0:1775768069.811658:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768069.811660:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768069.811663:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768069.811665:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768069.811666:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768069.811668:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768069.811670:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768069.811672:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225286:225286:256:4294967295] 192.168.202.156@tcp LPNI seq info [225282:225282:8:4294967295] 00000400:00000200:1.0:1775768069.811675:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768069.811678:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768069.811681:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768069.811683:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000800:00000200:1.0:1775768069.811685:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768069.811687:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775768069.811689:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768069.811691:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1775768069.811693:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768069.811695:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768069.811697:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768069.811699:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768069.811700:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768069.811701:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768069.811702:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768069.811704:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fe680 x1862022294270464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768094 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768069.811709:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768069.811711:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fe680 x1862022294270464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768094 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768069.811715:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768069.811717:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768069.811719:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768069.811721:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:1.0:1775768069.811723:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768069.811724:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299994659 00000100:00000001:1.0:1775768069.811726:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299995966 00000100:00000001:1.0:1775768069.811727:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299994659 00000100:00000001:1.0:1775768069.811728:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299995966 00000100:00000001:1.0:1775768069.811730:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768069.811731:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768069.811732:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768069.811734:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768069/real 1775768069] req@ffff9942099fe680 x1862022294270464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768069.811738:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768069.811739:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768069.811740:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768069.811741:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768069.811742:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768069.811744:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768069.811746:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fe680 x1862022294270464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768069.811751:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fe680 x1862022294270464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768069.811754:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768069.811755:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768069.811756:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768069.811757:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768069.811759:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fe680 x1862022294270464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768069.811763:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768069.811765:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fe680 x1862022294270464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768094 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768069.811770:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768069.811771:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768069.811772:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768069.811773:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768069.811774:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768069.811777:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768069.811778:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768069.811780:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768069.811782:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768069.811784:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fe680 x1862022294270464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768094 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768069.811788:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294270464:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768069.811791:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768069.811792:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768069.811794:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fce00 x1862022294270208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768094 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768069.811798:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768069.811800:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768069.811801:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543e800. 02000000:00000001:1.0:1775768069.811804:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768069.811806:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768069.811807:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:1.0:1775768069.811809:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768069.811811:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029400. 02000000:00000001:1.0:1775768069.811814:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768069.811816:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768069.811817:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fce00. 00000100:00000001:1.0:1775768069.811820:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768069.811821:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768069.811823:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768069.811827:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fdc00 x1862022294270336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768094 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768069.811835:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768069.811836:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768069.811838:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543c800. 02000000:00000001:1.0:1775768069.811840:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768069.811841:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768069.811842:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:1.0:1775768069.811844:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768069.811845:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002bc00. 02000000:00000001:1.0:1775768069.811848:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768069.811849:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768069.811850:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fdc00. 00000100:00000001:1.0:1775768069.811852:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768069.811853:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768069.811854:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768069.811858:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fe680 x1862022294270464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768094 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768069.811864:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768069.811865:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768069.811866:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543c400. 02000000:00000001:1.0:1775768069.811868:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768069.811869:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768069.811870:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:1.0:1775768069.811872:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768069.811873:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002a800. 02000000:00000001:1.0:1775768069.811876:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768069.811877:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768069.811878:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fe680. 00000100:00000001:1.0:1775768069.811880:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768069.811881:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768069.811882:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768069.811886:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768069.811888:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768069.811890:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768069.811892:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768069.811893:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768069.811898:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fca80 x1862022294270592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768069.811906:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768069.811907:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768069.811909:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768069.811911:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768069.811912:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768069.811915:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294270592:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768069.811917:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768069.811919:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768069.811920:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768069.811921:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768069.811923:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543c400. 02000000:00000001:1.0:1775768069.811925:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768069.811927:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768069.811929:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000400:00000010:1.0:1775768069.811931:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000200:1.0:1775768069.811933:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294270592, portal 4 00000100:00000001:1.0:1775768069.811934:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768069.811935:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641199232 : -112966068352384 : ffff9942099fca80) 00000100:00000040:1.0:1775768069.811939:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fca80 x1862022294270592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768094 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768069.811943:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768069.811945:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768069.811946:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768069.811948:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294270592, offset 0 00000400:00000200:1.0:1775768069.811950:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768069.811953:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768069.811957:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768069.811960:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768069.811963:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768069.811966:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768069.811968:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768069.811971:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225287:225287:256:4294967295] 192.168.202.156@tcp LPNI seq info [225283:225283:8:4294967295] 00000400:00000200:1.0:1775768069.811975:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768069.811981:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768069.811986:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768069.811990:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000800:00000200:1.0:1775768069.811994:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768069.811997:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775768069.812001:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768069.812004:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1775768069.812006:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768069.812008:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768069.812010:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768069.812012:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768069.812014:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768069.812015:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768069.812017:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768069.812020:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fca80 x1862022294270592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768094 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768069.812025:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768069.812027:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fca80 x1862022294270592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768094 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768069.812031:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768069.812032:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768069.812035:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768069.812037:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:1.0:1775768069.812039:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768069.812041:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299994662 00000100:00000001:1.0:1775768069.812043:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4299995966 00000100:00000001:1.0:1775768069.812044:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299994662 00000100:00000001:1.0:1775768069.812047:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4299995966 00000100:00000001:1.0:1775768069.812048:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768069.812050:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768069.812051:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768069.812054:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768069/real 1775768069] req@ffff9942099fca80 x1862022294270592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768069.812059:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768069.812060:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768069.812061:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768069.812062:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768069.812063:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768069.812065:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768069.812067:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fca80 x1862022294270592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768069.812072:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fca80 x1862022294270592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768069.812076:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768069.812077:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768069.812078:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768069.812079:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768069.812081:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fca80 x1862022294270592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768069.812084:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768069.812087:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fca80 x1862022294270592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768094 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768069.812091:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768069.812092:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768069.812093:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768069.812094:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768069.812096:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768069.812099:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768069.812100:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768069.812103:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768069.812104:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768069.812107:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fca80 x1862022294270592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768094 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768069.812112:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294270592:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768069.812114:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768069.812115:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768069.812117:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fca80 x1862022294270592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768094 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768069.812121:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768069.812121:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768069.812123:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543c400. 02000000:00000001:1.0:1775768069.812124:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768069.812125:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768069.812125:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:1.0:1775768069.812126:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768069.812127:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029800. 02000000:00000001:1.0:1775768069.812129:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768069.812130:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768069.812130:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fca80. 00000100:00000001:1.0:1775768069.812132:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768069.812133:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768069.812134:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768069.812136:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768069.812137:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768069.812138:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768069.812139:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768069.873327:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768069.873333:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768069.873339:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768069.873340:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768069.873343:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768069.873344:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768069.873347:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768069.873347:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768069.873350:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768069.873351:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768069.873374:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768069.873377:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768069.873379:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768069.937351:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768069.937369:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768069.937372:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768069.937382:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768069.937388:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768069.937393:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768069.937396:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768069.937399:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768069.937401:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768070.833320:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768070.833329:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768070.833331:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768070.833333:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:3.0:1775768070.833338:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775768070.833339:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768070.833344:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768070.833344:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:0.0:1775768070.833350:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775768070.833350:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768070.833356:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768070.833356:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768070.833360:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768070.833361:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768070.833365:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768070.833365:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768070.833370:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768070.833374:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768070.833385:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768070.833388:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768070.833390:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768070.833393:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768070.833395:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768070.833398:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768070.833400:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768070.833402:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768070.833405:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768070.897247:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768070.897251:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768070.897255:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768070.897256:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768070.897260:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768070.897261:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768070.897264:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768070.897265:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768070.897268:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768070.897268:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768070.897315:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768070.897318:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768070.897322:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768070.961267:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768070.961273:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768070.961275:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768070.961280:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768070.961284:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768070.961288:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768070.961291:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768070.961294:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768070.961298:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768071.153270:0:14953:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4299995966, at 4299995968 00000100:00000040:1.0:1775768071.153270:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4299995966, at 4299995968 00000100:00000010:1.0:1775768071.153285:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cba840. 00000100:00000010:0.0:1775768071.153288:0:14953:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cba8a0. 00000100:00000001:0.0:1775768071.857281:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768071.857281:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768071.857288:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768071.857289:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768071.857292:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768071.857292:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768071.857293:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768071.857296:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768071.857299:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768071.857299:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:0.0:1775768071.857300:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775768071.857304:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:2.0:1775768071.857305:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775768071.857306:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768071.857308:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768071.857311:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768071.857311:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768071.857312:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768071.857315:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768071.857316:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768071.857319:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768071.857324:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768071.857327:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768071.857330:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768071.857330:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768071.857334:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768071.857337:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768071.921282:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768071.921287:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768071.921288:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768071.921291:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768071.921295:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768071.921296:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768071.921298:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768071.921299:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768071.921302:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768071.921303:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768071.921306:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768071.921306:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768071.921309:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768071.921309:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768071.921312:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768071.921312:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768071.921333:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768071.921335:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768071.921340:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768071.985290:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768071.985295:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768071.985297:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768071.985302:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768071.985304:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768071.985308:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768071.985310:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768071.985313:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768071.985315:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768072.881278:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768072.881278:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768072.881284:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768072.881285:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768072.881287:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768072.881288:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768072.881292:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:2.0:1775768072.881294:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768072.881299:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768072.881303:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768072.881304:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768072.881307:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768072.881308:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768072.881309:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768072.881313:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768072.881313:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768072.881314:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768072.881319:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768072.881320:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768072.881320:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768072.881321:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768072.881324:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768072.881325:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768072.881329:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768072.881329:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768072.881331:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768072.881333:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768072.945293:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768072.945297:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768072.945301:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768072.945302:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768072.945305:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768072.945306:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768072.945308:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768072.945309:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768072.945311:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768072.945312:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768072.945382:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768072.945387:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768072.945390:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768073.009258:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768073.009264:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768073.009266:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768073.009270:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768073.009273:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768073.009278:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768073.009282:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768073.009284:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768073.009287:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768073.905274:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768073.905275:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768073.905275:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768073.905280:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768073.905282:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768073.905283:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768073.905283:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768073.905288:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768073.905288:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768073.905289:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:1.0:1775768073.905295:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768073.905295:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1775768073.905295:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775768073.905300:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768073.905301:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768073.905302:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768073.905303:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768073.905306:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768073.905308:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768073.905308:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768073.905309:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768073.905315:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768073.905318:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768073.905322:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768073.905324:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768073.905328:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768073.905331:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768073.969285:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768073.969290:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768073.969294:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768073.969295:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768073.969298:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768073.969300:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768073.969302:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768073.969302:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768073.969305:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768073.969305:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768073.969321:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768073.969323:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768073.969325:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768074.033269:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768074.033274:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768074.033276:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768074.033281:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768074.033284:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768074.033287:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768074.033289:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768074.033291:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768074.033293:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768074.929250:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768074.929256:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768074.929261:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768074.929261:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768074.929263:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768074.929264:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768074.929270:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:2.0:1775768074.929272:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775768074.929272:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768074.929275:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768074.929277:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768074.929279:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768074.929279:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768074.929282:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768074.929283:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768074.929284:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775768074.929285:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768074.929289:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768074.929290:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768074.929292:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768074.929293:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768074.929294:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768074.929296:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768074.929300:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768074.929301:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768074.929305:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768074.929308:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768074.929416:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768074.929421:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775768074.929423:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768074.929426:0:36:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768074.929429:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768074.929434:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5327 00000400:00000200:0.0:1775768074.929442:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768074.929446:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768074.929449:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768074.929451:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768074.929454:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768074.929455:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768074.929457:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768074.929460:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768074.929461:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:0.0:1775768074.929463:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768074.929465:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:0.0:1775768074.929466:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:0.0:1775768074.929468:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:0.0:1775768074.929470:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768074.929471:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768074.929473:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768074.929474:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768074.929476:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:0.0:1775768074.929478:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768074.929480:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768074.929483:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768074.929485:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768074.929487:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768074.929489:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775768074.929490:0:36:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768074.929498:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fc380. 00000020:00000040:0.0:1775768074.929500:0:36:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:0.0:1775768074.929503:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768074.929504:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768074.929506:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768074.929509:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768074.929513:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768074.929520:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002b800. 00000100:00000001:0.0:1775768074.929525:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768074.929532:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fc380 x1862022294270720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768074.929540:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768074.929543:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768074.929547:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002957419c] to pc [ptlrpcd_rcv+-1] req@ffff9942099fc380 x1862022294270720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768074.929564:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768074.929566:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768074.929569:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775768074.929570:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768074.929571:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768074.929573:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768074.929574:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5327 00000400:00000200:0.0:1775768074.929576:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768074.929578:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768074.929580:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768074.929583:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768074.929585:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768074.929587:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768074.929589:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768074.929590:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768074.929591:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:0.0:1775768074.929592:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:1.0:1775768074.929593:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:0.0:1775768074.929594:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:0.0:1775768074.929598:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000001:1.0:1775768074.929598:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1775768074.929601:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000001:1.0:1775768074.929602:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768074.929605:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768074.929606:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768074.929610:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000040:1.0:1775768074.929610:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fc380 x1862022294270720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768074.929614:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768074.929619:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00080000:0.0:1775768074.929626:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775768074.929627:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768074.929629:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768074.929631:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768074.929633:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768074.929635:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775768074.929636:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768074.929639:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768074.929642:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1775768074.929643:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294270720:192.168.202.156@tcp:250:kworker.0 02000000:00000001:0.0:1775768074.929648:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768074.929649:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1775768074.929652:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:1.0:1775768074.929653:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768074.929656:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1775768074.929657:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1775768074.929658:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:0.0:1775768074.929663:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775768074.929664:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543c400. 00000100:00000001:0.0:1775768074.929665:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768074.929667:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768074.929672:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fed80. 00000400:00000200:1.0:1775768074.929673:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000040:0.0:1775768074.929677:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000400:00000010:1.0:1775768074.929679:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 02000000:00000001:0.0:1775768074.929680:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768074.929682:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768074.929684:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000010:1.0:1775768074.929684:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 02000000:00000001:0.0:1775768074.929685:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768074.929688:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775768074.929692:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294270720, portal 25 02000000:00000010:0.0:1775768074.929693:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220028400. 00000100:00000001:1.0:1775768074.929697:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768074.929698:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768074.929699:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641197440 : -112966068354176 : ffff9942099fc380) 00000100:00100000:0.0:1775768074.929707:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fed80 x1862022294270848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:1.0:1775768074.929708:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fc380 x1862022294270720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768099 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768074.929737:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1775768074.929738:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:1.0:1775768074.929741:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000100:00000001:0.0:1775768074.929744:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775768074.929745:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768074.929749:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294270720, offset 0 00000100:00000040:0.0:1775768074.929753:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000c09b4d8] to pc [ptlrpcd_rcv+-1] req@ffff9942099fed80 x1862022294270848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775768074.929758:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768074.929768:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768074.929771:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768074.929775:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00080000:0.0:1775768074.929776:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1775768074.929780:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:0.0:1775768074.929783:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775768074.929785:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:1.0:1775768074.929787:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000100:00080000:0.0:1775768074.929788:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768074.929792:0:36:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:1.0:1775768074.929795:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000100:00080000:0.0:1775768074.929796:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5327 00000400:00000200:1.0:1775768074.929803:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768074.929808:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225288:225288:256:4294967295] 192.168.202.156@tcp LPNI seq info [225284:225284:8:4294967295] 00000400:00000200:1.0:1775768074.929814:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768074.929825:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768074.929832:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768074.929838:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000800:00000200:1.0:1775768074.929857:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768074.929860:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775768074.929865:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768074.929867:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000100:1.0:1775768074.929870:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768074.929872:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768074.929877:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768074.929878:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768074.929884:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000400:00000200:1.0:1775768074.929885:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000100:00080000:0.0:1775768074.929888:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768074.929892:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000200:1.0:1775768074.929892:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768074.929896:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:0.0:1775768074.929897:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768074.929901:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768074.929902:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1775768074.929903:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768074.929906:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768074.929908:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000100:00000200:1.0:1775768074.929909:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fc380 x1862022294270720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768099 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775768074.929910:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768074.929912:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:0.0:1775768074.929915:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:0.0:1775768074.929919:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:0.0:1775768074.929924:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768074.929926:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775768074.929927:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768074.929930:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000040:1.0:1775768074.929933:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fc380 x1862022294270720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768099 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768074.929934:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768074.929938:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:0.0:1775768074.929943:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768074.929947:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768074.929948:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768074.929949:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1775768074.929952:0:2399:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:0.0:1775768074.929955:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768074.929958:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768074.929959:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768074.929963:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:1.0:1775768074.929965:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00100000:0.0:1775768074.929967:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1775768074.929968:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4300001085 00000008:00000001:0.0:1775768074.929971:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775768074.929974:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba840. 00000100:00000001:0.0:1775768074.929975:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768074.929981:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fd180. 00000100:00000001:1.0:1775768074.929982:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000020:00000040:0.0:1775768074.929986:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000100:00000001:1.0:1775768074.929989:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4300001085 02000000:00000001:0.0:1775768074.929990:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768074.929993:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775768074.929993:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cbaf60. 00000100:00000001:0.0:1775768074.929996:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1775768074.929997:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 02000000:00000001:0.0:1775768074.929999:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1775768074.930001:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768074.930002:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768074.930007:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768074.930008:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220029800. 00000100:00000001:1.0:1775768074.930011:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1775768074.930012:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768074.930019:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768074/real 1775768074] req@ffff9942099fc380 x1862022294270720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768074.930020:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fd180 x1862022294270976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768074.930038:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768074.930041:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768074.930049:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004ec16d17] to pc [ptlrpcd_rcv+-1] req@ffff9942099fd180 x1862022294270976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768074.930049:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768074.930052:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768074.930056:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768074.930062:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000100:00000001:0.0:1775768074.930064:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775768074.930065:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00080000:0.0:1775768074.930070:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775768074.930071:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00080000:0.0:1775768074.930076:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000200:1.0:1775768074.930077:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fc380 x1862022294270720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768074.930080:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768074.930083:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768074.930089:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00000200:1.0:1775768074.930090:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fc380 x1862022294270720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768074.930094:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5327 00000400:00000200:0.0:1775768074.930103:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768074.930107:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:0.0:1775768074.930108:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768074.930112:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1775768074.930112:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768074.930117:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:0.0:1775768074.930118:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768074.930121:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768074.930122:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768074.930127:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00100000:1.0:1775768074.930130:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fc380 x1862022294270720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768074.930131:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768074.930135:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768074.930138:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:0.0:1775768074.930142:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:1.0:1775768074.930146:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:0.0:1775768074.930147:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:0.0:1775768074.930151:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:1.0:1775768074.930152:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fc380 x1862022294270720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768099 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768074.930189:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:0.0:1775768074.930193:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:1.0:1775768074.930193:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768074.930196:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768074.930197:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768074.930199:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768074.930201:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:0.0:1775768074.930202:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768074.930207:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00080000:1.0:1775768074.930207:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000040:0.0:1775768074.930212:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00080000:1.0:1775768074.930215:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768074.930216:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:1.0:1775768074.930218:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000040:0.0:1775768074.930221:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00000040:1.0:1775768074.930224:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fc380 x1862022294270720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768099 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768074.930228:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768074.930235:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768074.930239:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:1.0:1775768074.930239:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294270720:192.168.202.156@tcp:250:kworker.0 02000000:00000001:0.0:1775768074.930244:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768074.930244:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768074.930246:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1775768074.930250:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768074.930250:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fc380 x1862022294270720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768099 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768074.930255:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775768074.930259:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000100:00000001:1.0:1775768074.930260:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768074.930263:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000008:00000001:0.0:1775768074.930264:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775768074.930266:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543c400. 00000100:00000001:0.0:1775768074.930268:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768074.930269:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768074.930272:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768074.930273:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000100:00000010:0.0:1775768074.930274:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ff480. 00000020:00000001:1.0:1775768074.930275:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768074.930278:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002b800. 00000020:00000040:0.0:1775768074.930279:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:1.0:1775768074.930282:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768074.930283:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768074.930286:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768074.930286:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1775768074.930289:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000010:1.0:1775768074.930289:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fc380. 02000000:00000001:0.0:1775768074.930292:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1775768074.930293:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 02000000:00000001:0.0:1775768074.930294:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768074.930295:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768074.930298:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:0.0:1775768074.930299:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002a800. 00000100:00000001:0.0:1775768074.930302:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768074.930304:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768074.930306:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768074.930308:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ff480 x1862022294271104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775768074.930309:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768074.930311:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768074.930314:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768074.930320:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fed80 x1862022294270848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768074.930321:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768074.930325:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768074.930332:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e73dcd12] to pc [ptlrpcd_rcv+-1] req@ffff9942099ff480 x1862022294271104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768074.930337:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768074.930340:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768074.930343:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768074.930343:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768074.930345:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:0.0:1775768074.930348:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1775768074.930348:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768074.930354:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294270848:192.168.202.156@tcp:38:kworker.0 00000100:00080000:0.0:1775768074.930355:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775768074.930359:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775768074.930360:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00080000:0.0:1775768074.930363:0:36:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:1.0:1775768074.930363:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768074.930365:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768074.930368:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1775768074.930369:0:36:0:(import.c:484:import_select_connection()) Process entered 02000000:00000010:1.0:1775768074.930371:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543c400. 00000100:00080000:0.0:1775768074.930374:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5327 02000000:00000001:1.0:1775768074.930375:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768074.930381:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768074.930385:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768074.930385:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000400:00000200:0.0:1775768074.930392:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000400:00000010:1.0:1775768074.930394:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00080000:0.0:1775768074.930395:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000200:1.0:1775768074.930399:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294270848, portal 10 00000100:00000040:0.0:1775768074.930400:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768074.930404:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775768074.930404:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768074.930407:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641208192 : -112966068343424 : ffff9942099fed80) 00000100:00000040:0.0:1775768074.930408:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768074.930412:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768074.930414:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768074.930416:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000100:00000040:1.0:1775768074.930417:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fed80 x1862022294270848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768099 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775768074.930419:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768074.930422:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:0.0:1775768074.930426:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:0.0:1775768074.930430:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000001:1.0:1775768074.930432:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768074.930435:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000040:1.0:1775768074.930438:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000100:00000001:0.0:1775768074.930439:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:1.0:1775768074.930441:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000040:0.0:1775768074.930443:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000200:1.0:1775768074.930445:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294270848, offset 0 00000100:00000001:0.0:1775768074.930446:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768074.930450:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000400:00000200:1.0:1775768074.930453:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00080000:0.0:1775768074.930455:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768074.930457:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768074.930461:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1775768074.930461:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:0.0:1775768074.930465:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1775768074.930467:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 02000000:00000001:0.0:1775768074.930469:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768074.930471:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1775768074.930472:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:1.0:1775768074.930475:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000008:00100000:0.0:1775768074.930476:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000400:00000200:1.0:1775768074.930479:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000008:00000001:0.0:1775768074.930480:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768074.930484:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768074.930484:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00000010:0.0:1775768074.930490:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fd880. 00000400:00000200:1.0:1775768074.930490:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225289:225289:256:4294967295] 192.168.202.156@tcp LPNI seq info [225285:225285:8:4294967295] 00000020:00000040:0.0:1775768074.930494:0:36:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00000400:00000200:1.0:1775768074.930498:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:0.0:1775768074.930499:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768074.930502:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768074.930505:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768074.930508:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768074.930511:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768074.930511:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 02000000:00000010:0.0:1775768074.930517:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002bc00. 00000800:00000200:1.0:1775768074.930518:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00000001:0.0:1775768074.930521:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1775768074.930523:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000100:00100000:0.0:1775768074.930527:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fd880 x1862022294271232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.0:1775768074.930530:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768074.930535:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000100:00000001:0.0:1775768074.930544:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000200:1.1:1775768074.930545:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:0.0:1775768074.930549:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775768074.930549:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000100:00000040:0.0:1775768074.930555:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007f8a5f3b] to pc [ptlrpcd_rcv+-1] req@ffff9942099fd880 x1862022294271232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:1.0:1775768074.930556:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768074.930559:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768074.930563:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:0.0:1775768074.930570:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768074.930571:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768074.930576:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000100:00000040:0.0:1775768074.930577:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5337) 00000400:00000010:1.0:1775768074.930577:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768074.930582:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768074.930585:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fed80 x1862022294270848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768099 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768074.930591:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768074.930593:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fed80 x1862022294270848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768099 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768074.930597:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768074.930598:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768074.930600:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768074.930603:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:1.0:1775768074.930605:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768074.930606:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299999763 00000100:00000001:1.0:1775768074.930608:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300001085 00000100:00000001:1.0:1775768074.930610:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299999763 00000100:00000001:1.0:1775768074.930611:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300001085 00000100:00000001:1.0:1775768074.930613:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768074.930614:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768074.930615:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768074.930618:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768074/real 1775768074] req@ffff9942099fed80 x1862022294270848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768074.930627:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768074.930628:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768074.930629:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768074.930630:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768074.930631:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768074.930633:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768074.930634:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fed80 x1862022294270848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768074.930640:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fed80 x1862022294270848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768074.930644:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768074.930645:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768074.930646:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768074.930647:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768074.930649:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fed80 x1862022294270848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768074.930653:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768074.930655:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fed80 x1862022294270848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768099 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768074.930659:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768074.930661:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768074.930662:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768074.930663:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768074.930664:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768074.930666:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768074.930667:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768074.930669:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768074.930671:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768074.930673:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fed80 x1862022294270848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768099 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768074.930679:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294270848:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768074.930681:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768074.930683:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fd180 x1862022294270976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768074.930687:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768074.930688:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768074.930689:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768074.930690:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768074.930692:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768074.930693:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294270976:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768074.930695:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768074.930696:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768074.930697:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768074.930698:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768074.930700:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543c800. 02000000:00000001:1.0:1775768074.930701:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768074.930703:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768074.930705:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000400:00000010:1.0:1775768074.930706:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000200:1.0:1775768074.930708:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294270976, portal 10 00000100:00000001:1.0:1775768074.930709:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768074.930710:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641201024 : -112966068350592 : ffff9942099fd180) 00000100:00000040:1.0:1775768074.930714:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fd180 x1862022294270976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768099 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768074.930718:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768074.930720:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768074.930721:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768074.930722:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294270976, offset 0 00000400:00000200:1.0:1775768074.930725:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768074.930727:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768074.930729:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768074.930731:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768074.930733:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768074.930735:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768074.930736:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768074.930738:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225290:225290:256:4294967295] 192.168.202.156@tcp LPNI seq info [225286:225286:8:4294967295] 00000400:00000200:1.0:1775768074.930741:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768074.930745:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768074.930748:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768074.930750:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000800:00000200:1.0:1775768074.930752:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768074.930754:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775768074.930757:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768074.930770:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000100:1.0:1775768074.930772:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768074.930774:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768074.930776:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768074.930778:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768074.930781:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768074.930782:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768074.930784:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768074.930788:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fd180 x1862022294270976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768099 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768074.930796:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768074.930799:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fd180 x1862022294270976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768099 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768074.930806:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768074.930807:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768074.930811:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768074.930814:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:1.0:1775768074.930818:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768074.930820:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299999755 00000100:00000001:1.0:1775768074.930823:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300001085 00000100:00000001:1.0:1775768074.930825:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299999755 00000100:00000001:1.0:1775768074.930827:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300001085 00000100:00000001:1.0:1775768074.930830:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768074.930832:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768074.930834:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768074.930838:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768074/real 1775768074] req@ffff9942099fd180 x1862022294270976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768074.930847:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768074.930848:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768074.930850:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768074.930852:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768074.930854:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768074.930856:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768074.930858:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fd180 x1862022294270976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768074.930864:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fd180 x1862022294270976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768074.930868:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768074.930869:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768074.930870:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768074.930870:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768074.930872:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fd180 x1862022294270976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768074.930877:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768074.930879:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fd180 x1862022294270976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768099 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768074.930884:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768074.930886:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768074.930887:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768074.930888:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768074.930889:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768074.930891:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768074.930893:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768074.930895:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768074.930897:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768074.930899:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fd180 x1862022294270976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768099 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768074.930903:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294270976:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768074.930906:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768074.930907:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768074.930909:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fed80 x1862022294270848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768099 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768074.930912:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768074.930913:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768074.930915:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543c400. 02000000:00000001:1.0:1775768074.930916:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768074.930917:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768074.930918:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:1.0:1775768074.930920:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768074.930921:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028400. 02000000:00000001:1.0:1775768074.930923:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768074.930924:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768074.930925:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fed80. 00000100:00000001:1.0:1775768074.930927:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768074.930928:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768074.930929:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768074.930931:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fd180 x1862022294270976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768099 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768074.930939:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768074.930940:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768074.930941:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543c800. 02000000:00000001:1.0:1775768074.930942:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768074.930943:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768074.930943:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:1.0:1775768074.930944:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768074.930945:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029800. 02000000:00000001:1.0:1775768074.930946:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768074.930947:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768074.930948:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fd180. 00000100:00000001:1.0:1775768074.930949:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768074.930950:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768074.930951:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768074.930953:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768074.930954:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768074.930955:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768074.930956:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768074.930957:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768074.930959:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ff480 x1862022294271104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768074.930963:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768074.930964:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768074.930965:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768074.930966:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768074.930968:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768074.930969:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294271104:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768074.930971:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768074.930973:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768074.930974:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768074.930975:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768074.930977:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543c800. 02000000:00000001:1.0:1775768074.930978:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768074.930981:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768074.930983:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000400:00000010:1.0:1775768074.930985:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000200:1.0:1775768074.930987:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294271104, portal 4 00000100:00000001:1.0:1775768074.930988:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768074.930989:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641209984 : -112966068341632 : ffff9942099ff480) 00000100:00000040:1.0:1775768074.930993:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ff480 x1862022294271104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768099 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768074.930997:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768074.930999:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768074.931000:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768074.931002:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294271104, offset 0 00000400:00000200:1.0:1775768074.931006:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768074.931010:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768074.931013:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768074.931016:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768074.931019:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768074.931021:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768074.931023:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768074.931026:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225291:225291:256:4294967295] 192.168.202.156@tcp LPNI seq info [225287:225287:8:4294967295] 00000400:00000200:1.0:1775768074.931031:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768074.931037:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768074.931042:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768074.931045:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000800:00000200:1.0:1775768074.931049:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768074.931052:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775768074.931057:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768074.931060:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000100:1.0:1775768074.931064:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768074.931066:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768074.931069:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768074.931072:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768074.931074:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768074.931076:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768074.931078:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768074.931080:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ff480 x1862022294271104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768099 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768074.931086:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768074.931088:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ff480 x1862022294271104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768099 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768074.931095:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768074.931096:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768074.931100:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768074.931103:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:1.0:1775768074.931106:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768074.931108:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299999778 00000100:00000001:1.0:1775768074.931110:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300001085 00000100:00000001:1.0:1775768074.931113:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299999778 00000100:00000001:1.0:1775768074.931115:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300001085 00000100:00000001:1.0:1775768074.931117:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768074.931119:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768074.931121:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768074.931126:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768074/real 1775768074] req@ffff9942099ff480 x1862022294271104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768074.931135:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768074.931136:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768074.931138:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768074.931140:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768074.931142:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768074.931144:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768074.931147:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ff480 x1862022294271104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768074.931152:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ff480 x1862022294271104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768074.931188:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768074.931189:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768074.931190:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768074.931191:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768074.931194:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ff480 x1862022294271104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768074.931199:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768074.931202:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ff480 x1862022294271104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768099 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768074.931206:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768074.931208:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768074.931208:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768074.931209:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768074.931210:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768074.931213:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768074.931214:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768074.931216:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768074.931217:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768074.931220:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ff480 x1862022294271104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768099 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768074.931224:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294271104:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768074.931227:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768074.931229:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fd880 x1862022294271232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768074.931234:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768074.931235:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768074.931237:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768074.931238:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768074.931239:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768074.931241:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294271232:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768074.931243:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768074.931244:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768074.931245:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768074.931247:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768074.931249:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543c400. 02000000:00000001:1.0:1775768074.931250:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768074.931253:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768074.931255:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000400:00000010:1.0:1775768074.931257:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000200:1.0:1775768074.931259:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294271232, portal 4 00000100:00000001:1.0:1775768074.931260:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768074.931261:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641202816 : -112966068348800 : ffff9942099fd880) 00000100:00000040:1.0:1775768074.931265:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fd880 x1862022294271232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768099 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768074.931269:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768074.931270:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768074.931272:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768074.931273:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294271232, offset 0 00000400:00000200:1.0:1775768074.931276:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768074.931278:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768074.931281:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768074.931282:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768074.931284:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768074.931286:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768074.931287:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768074.931289:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225292:225292:256:4294967295] 192.168.202.156@tcp LPNI seq info [225288:225288:8:4294967295] 00000400:00000200:1.0:1775768074.931292:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768074.931296:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768074.931300:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768074.931302:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000800:00000200:1.0:1775768074.931304:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768074.931306:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775768074.931309:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768074.931310:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000100:1.0:1775768074.931313:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768074.931315:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768074.931317:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768074.931319:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768074.931321:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768074.931322:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768074.931324:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768074.931326:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fd880 x1862022294271232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768099 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768074.931330:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768074.931332:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fd880 x1862022294271232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768099 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768074.931336:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768074.931337:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768074.931339:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768074.931341:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:1.0:1775768074.931343:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768074.931344:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4299999782 00000100:00000001:1.0:1775768074.931346:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300001085 00000100:00000001:1.0:1775768074.931347:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4299999782 00000100:00000001:1.0:1775768074.931348:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300001085 00000100:00000001:1.0:1775768074.931350:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768074.931351:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768074.931352:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768074.931354:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768074/real 1775768074] req@ffff9942099fd880 x1862022294271232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768074.931360:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768074.931360:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768074.931362:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768074.931363:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768074.931364:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768074.931365:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768074.931367:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fd880 x1862022294271232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768074.931371:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fd880 x1862022294271232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768074.931376:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768074.931377:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768074.931378:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768074.931379:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768074.931381:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fd880 x1862022294271232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768074.931385:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768074.931387:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fd880 x1862022294271232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768099 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768074.931391:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768074.931392:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768074.931393:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768074.931394:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768074.931395:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768074.931397:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768074.931398:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768074.931400:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768074.931401:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768074.931404:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fd880 x1862022294271232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768099 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768074.931409:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294271232:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768074.931412:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768074.931413:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768074.931415:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ff480 x1862022294271104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768099 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768074.931418:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768074.931419:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768074.931420:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543c800. 02000000:00000001:1.0:1775768074.931422:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768074.931422:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768074.931423:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:1.0:1775768074.931424:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768074.931425:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002a800. 02000000:00000001:1.0:1775768074.931427:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768074.931428:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768074.931428:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ff480. 00000100:00000001:1.0:1775768074.931430:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768074.931431:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768074.931432:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768074.931433:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fd880 x1862022294271232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768099 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768074.931437:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768074.931438:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768074.931439:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543c400. 02000000:00000001:1.0:1775768074.931440:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768074.931441:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768074.931441:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:1.0:1775768074.931442:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768074.931443:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002bc00. 02000000:00000001:1.0:1775768074.931444:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768074.931445:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768074.931446:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fd880. 00000100:00000001:1.0:1775768074.931447:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768074.931449:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768074.931450:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768074.931452:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768074.931453:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768074.931454:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768074.931455:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768074.993283:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768074.993288:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768074.993293:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768074.993294:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768074.993297:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768074.993298:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768074.993300:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768074.993300:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768074.993302:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768074.993303:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768074.993336:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768074.993339:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768074.993342:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768075.057219:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768075.057224:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768075.057226:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768075.057230:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768075.057234:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768075.057237:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768075.057239:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768075.057242:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768075.057244:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768075.953276:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768075.953276:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768075.953283:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768075.953283:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768075.953285:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768075.953285:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768075.953291:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:2.0:1775768075.953291:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768075.953295:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768075.953295:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768075.953299:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768075.953299:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768075.953302:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768075.953305:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768075.953307:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768075.953308:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768075.953310:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768075.953311:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768075.953388:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768075.953391:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768075.953392:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768075.953394:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768075.953395:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768075.953397:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768075.953398:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768075.953400:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768075.953401:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768076.017279:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768076.017285:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768076.017291:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768076.017294:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768076.017296:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768076.017300:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768076.017303:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768076.017305:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768076.017307:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768076.017310:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768076.017321:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768076.017322:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768076.017323:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768076.017325:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768076.017327:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768076.017329:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768076.017330:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768076.017332:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768076.017336:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768076.017337:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768076.017338:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768076.017340:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768076.017342:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768076.017343:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768076.017344:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768076.017346:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768076.017349:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768076.017351:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768076.017352:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768076.017353:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768076.017355:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768076.017356:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768076.017358:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768076.017360:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768076.017380:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768076.017383:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768076.017385:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768076.081251:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768076.081256:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768076.081258:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768076.081262:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768076.081265:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768076.081269:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768076.081271:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768076.081272:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768076.081275:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768076.273242:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4300001085, at 4300001088 00000100:00000040:0.0:1775768076.273253:0:14953:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4300001085, at 4300001088 00000100:00000010:1.0:1775768076.273258:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cbaf60. 00000100:00000010:0.0:1775768076.273261:0:14953:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cba840. 00000100:00000001:1.0:1775768076.977260:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768076.977260:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768076.977261:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768076.977267:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768076.977267:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768076.977268:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768076.977269:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768076.977270:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768076.977272:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768076.977275:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:2.0:1775768076.977276:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775768076.977277:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768076.977281:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768076.977281:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768076.977283:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768076.977286:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768076.977286:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768076.977286:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768076.977291:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768076.977294:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768076.977297:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768076.977297:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768076.977299:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768076.977302:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768076.977303:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768076.977306:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768076.977309:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768077.041268:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768077.041275:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768077.041280:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768077.041281:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768077.041284:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768077.041285:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768077.041288:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768077.041288:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768077.041291:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768077.041292:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768077.041350:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768077.041354:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768077.041358:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768077.105222:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768077.105229:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768077.105231:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768077.105236:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768077.105240:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768077.105244:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768077.105247:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768077.105250:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768077.105253:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768078.001245:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768078.001245:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768078.001252:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768078.001253:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768078.001254:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768078.001256:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768078.001261:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775768078.001262:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768078.001265:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768078.001267:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768078.001269:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768078.001272:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768078.001273:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768078.001276:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768078.001280:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768078.001280:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768078.001282:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768078.001284:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768078.001287:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768078.001289:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768078.001289:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768078.001295:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775768078.001298:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768078.001301:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768078.001303:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768078.001304:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768078.001306:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768078.065246:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768078.065252:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768078.065262:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768078.065263:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768078.065267:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768078.065268:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768078.065272:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768078.065274:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768078.065277:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768078.065278:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768078.065299:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768078.065302:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768078.065305:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768078.129279:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768078.129286:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768078.129289:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768078.129294:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768078.129299:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768078.129303:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768078.129306:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768078.129308:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768078.129312:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768079.025259:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768079.025259:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768079.025259:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768079.025266:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768079.025266:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768079.025267:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768079.025269:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768079.025270:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768079.025270:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768079.025277:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:1.0:1775768079.025277:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775768079.025278:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:0.0:1775768079.025283:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768079.025283:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768079.025284:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768079.025288:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768079.025288:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768079.025289:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768079.025293:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768079.025296:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768079.025298:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768079.025316:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768079.025319:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768079.025321:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768079.025322:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768079.025326:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768079.025329:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768079.089258:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768079.089262:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768079.089267:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768079.089268:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768079.089271:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768079.089272:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768079.089282:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768079.089283:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768079.089286:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768079.089287:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768079.089342:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768079.089345:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768079.089347:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768079.153230:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768079.153235:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768079.153236:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768079.153241:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768079.153245:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768079.153249:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768079.153252:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768079.153254:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768079.153257:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768080.049212:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768080.049218:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768080.049218:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768080.049222:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768080.049224:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:1.0:1775768080.049228:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775768080.049228:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768080.049234:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1775768080.049234:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768080.049237:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768080.049239:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768080.049241:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768080.049243:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768080.049244:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768080.049251:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768080.049251:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1775768080.049252:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1775768080.049255:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768080.049258:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768080.049259:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775768080.049263:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1775768080.049265:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:0.0:1775768080.049266:0:36:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775768080.049271:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768080.049272:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775768080.049275:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768080.049279:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00080000:0.0:1775768080.049281:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5332 00000100:00000001:2.0:1775768080.049285:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768080.049287:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768080.049290:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768080.049291:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768080.049295:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768080.049305:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768080.049312:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768080.049315:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768080.049318:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768080.049320:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768080.049323:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768080.049324:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768080.049327:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768080.049329:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:0.0:1775768080.049331:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768080.049333:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:0.0:1775768080.049334:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:0.0:1775768080.049336:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:0.0:1775768080.049338:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768080.049339:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768080.049340:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768080.049342:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768080.049344:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:0.0:1775768080.049346:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768080.049348:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768080.049350:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768080.049352:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768080.049355:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768080.049356:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775768080.049358:0:36:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768080.049365:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fe300. 00000020:00000040:0.0:1775768080.049368:0:36:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:0.0:1775768080.049371:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768080.049372:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768080.049375:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768080.049377:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768080.049381:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768080.049387:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220028000. 00000100:00000001:0.0:1775768080.049392:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768080.049399:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fe300 x1862022294271360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768080.049407:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768080.049410:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768080.049414:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008a1e3639] to pc [ptlrpcd_rcv+-1] req@ffff9942099fe300 x1862022294271360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768080.049430:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768080.049432:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768080.049435:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775768080.049436:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768080.049437:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768080.049439:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768080.049441:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5332 00000400:00000200:0.0:1775768080.049444:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768080.049447:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768080.049449:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768080.049451:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768080.049453:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775768080.049454:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1775768080.049455:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768080.049457:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768080.049458:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:1.0:1775768080.049460:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000001:0.0:1775768080.049462:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768080.049464:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:0.0:1775768080.049467:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000040:1.0:1775768080.049468:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fe300 x1862022294271360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1775768080.049469:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:0.0:1775768080.049474:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:0.0:1775768080.049479:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000001:1.0:1775768080.049485:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000040:0.0:1775768080.049487:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768080.049488:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768080.049493:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1775768080.049494:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:0.0:1775768080.049496:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000001:1.0:1775768080.049498:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768080.049499:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 02000000:00000001:1.0:1775768080.049502:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1775768080.049503:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00100000:1.0:1775768080.049510:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294271360:192.168.202.156@tcp:250:kworker.0 00000100:00080000:0.0:1775768080.049511:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768080.049516:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768080.049518:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768080.049519:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775768080.049523:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768080.049525:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768080.049526:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768080.049531:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768080.049532:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1775768080.049535:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000010:1.0:1775768080.049537:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543e800. 00000008:00100000:0.0:1775768080.049538:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1775768080.049541:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1775768080.049543:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768080.049545:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768080.049548:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fc700. 00000400:00000200:1.0:1775768080.049548:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000040:0.0:1775768080.049552:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000400:00000010:1.0:1775768080.049552:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 02000000:00000001:0.0:1775768080.049555:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768080.049557:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768080.049559:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000010:1.0:1775768080.049559:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 02000000:00000001:0.0:1775768080.049562:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768080.049564:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775768080.049567:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294271360, portal 25 00000100:00000001:1.0:1775768080.049569:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768080.049570:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641205504 : -112966068346112 : ffff9942099fe300) 00000100:00000040:1.0:1775768080.049576:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fe300 x1862022294271360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768105 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1775768080.049580:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220029400. 00000100:00000001:1.0:1775768080.049583:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1775768080.049586:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768080.049587:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768080.049591:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768080.049594:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294271360, offset 0 00000100:00100000:0.0:1775768080.049597:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fc700 x1862022294271488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775768080.049603:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00000001:0.0:1775768080.049612:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1775768080.049612:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768080.049617:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768080.049619:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00000040:0.0:1775768080.049626:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003283b20a] to pc [ptlrpcd_rcv+-1] req@ffff9942099fc700 x1862022294271488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775768080.049626:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768080.049630:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768080.049636:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768080.049641:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768080.049648:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225293:225293:256:4294967295] 192.168.202.156@tcp LPNI seq info [225289:225289:8:4294967295] 00000400:00000200:1.0:1775768080.049661:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768080.049666:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768080.049671:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1775768080.049675:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000100:00080000:0.0:1775768080.049676:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775768080.049680:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768080.049682:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768080.049687:0:36:0:(import.c:484:import_select_connection()) Process entered 00000800:00000200:1.0:1775768080.049688:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00080000:0.0:1775768080.049692:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5332 00000800:00000010:1.0:1775768080.049695:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000400:00000200:0.0:1775768080.049697:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768080.049703:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768080.049705:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000800:00000200:1.0:1775768080.049708:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000040:0.0:1775768080.049710:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768080.049714:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768080.049716:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000800:00000200:1.0:1775768080.049716:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000100:00000001:0.0:1775768080.049720:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768080.049723:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768080.049724:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:0.0:1775768080.049726:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768080.049729:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000800:00000200:1.1:1775768080.049730:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000001:0.0:1775768080.049731:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:0.0:1775768080.049736:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000800:00000200:1.1:1775768080.049737:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000100:00000040:0.0:1775768080.049740:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768080.049744:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768080.049747:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000800:00000100:1.0:1775768080.049749:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000100:00000001:0.0:1775768080.049750:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768080.049754:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000800:00000010:1.0:1775768080.049755:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000100:00080000:0.0:1775768080.049757:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768080.049760:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768080.049762:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1775768080.049764:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000001:0.0:1775768080.049765:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768080.049768:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768080.049772:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000100:00000001:0.0:1775768080.049773:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775768080.049776:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775768080.049779:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768080.049779:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000100:00000001:0.0:1775768080.049783:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775768080.049783:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768080.049788:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000010:0.0:1775768080.049790:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ff100. 00000100:00000200:1.0:1775768080.049794:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fe300 x1862022294271360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768105 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1775768080.049795:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000100:00000001:1.0:1775768080.049803:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768080.049807:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fe300 x1862022294271360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768105 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1775768080.049810:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768080.049813:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768080.049817:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768080.049818:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1775768080.049820:0:2399:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:0.0:1775768080.049822:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768080.049825:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768080.049827:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768080.049831:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002bc00. 00000400:00000200:1.0:1775768080.049832:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768080.049837:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768080.049837:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4300006205 00000100:00000010:1.0:1775768080.049843:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cbaf60. 00000100:00100000:0.0:1775768080.049845:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ff100 x1862022294271616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775768080.049849:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775768080.049853:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768080.049856:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768080.049856:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4300006205 00000100:00000010:1.0:1775768080.049860:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba180. 00000100:00000040:0.0:1775768080.049861:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5e24158] to pc [ptlrpcd_rcv+-1] req@ffff9942099ff100 x1862022294271616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768080.049864:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775768080.049868:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768080.049873:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768080.049875:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768080.049878:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00080000:0.0:1775768080.049879:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768080.049882:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775768080.049884:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:1.0:1775768080.049884:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768080/real 1775768080] req@ffff9942099fe300 x1862022294271360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768080.049887:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768080.049893:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768080.049898:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5332 00000400:00000001:1.0:1775768080.049899:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768080.049901:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1775768080.049903:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768080.049904:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:0.0:1775768080.049908:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000400:00000200:1.0:1775768080.049908:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768080.049911:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00080000:0.0:1775768080.049912:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1775768080.049915:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:0.0:1775768080.049918:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000200:1.0:1775768080.049921:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fe300 x1862022294271360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768080.049922:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768080.049927:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768080.049933:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768080.049936:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000200:1.0:1775768080.049937:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fe300 x1862022294271360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1775768080.049939:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:0.0:1775768080.049943:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768080.049946:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000100:00000001:1.0:1775768080.049947:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:0.0:1775768080.049949:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:1.0:1775768080.049950:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768080.049954:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:0.0:1775768080.049955:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:1.0:1775768080.049956:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768080.049960:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00100000:1.0:1775768080.049962:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fe300 x1862022294271360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768080.049963:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768080.049968:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768080.049972:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768080.049978:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00000001:1.0:1775768080.049979:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1775768080.049983:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000040:1.0:1775768080.049986:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fe300 x1862022294271360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768105 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768080.049987:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768080.049992:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768080.049996:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768080.049997:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:0.0:1775768080.050000:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768080.050001:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768080.050003:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775768080.050004:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000008:00100000:0.0:1775768080.050007:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000100:00000001:1.0:1775768080.050007:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768080.050009:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000008:00000001:0.0:1775768080.050010:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768080.050011:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768080.050013:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768080.050015:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000010:0.0:1775768080.050018:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fd500. 00000100:00080000:1.0:1775768080.050020:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000020:00000040:0.0:1775768080.050022:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00000100:00000001:1.0:1775768080.050023:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:0.0:1775768080.050026:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768080.050029:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768080.050029:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fe300 x1862022294271360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768105 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768080.050033:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768080.050036:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768080.050039:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768080.050043:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002a800. 00000100:00100000:1.0:1775768080.050044:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294271360:192.168.202.156@tcp:250:kworker.0 00000100:00000001:0.0:1775768080.050049:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768080.050050:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768080.050053:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00100000:0.0:1775768080.050055:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fd500 x1862022294271744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:1.0:1775768080.050058:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fe300 x1862022294271360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768105 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768080.050079:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768080.050081:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768080.050084:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543e800. 00000100:00000001:0.0:1775768080.050090:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1775768080.050090:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768080.050093:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:0.0:1775768080.050095:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1775768080.050096:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:1.0:1775768080.050100:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000040:0.0:1775768080.050104:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f246bb2b] to pc [ptlrpcd_rcv+-1] req@ffff9942099fd500 x1862022294271744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1775768080.050105:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028000. 02000000:00000001:1.0:1775768080.050111:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768080.050114:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768080.050118:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fe300. 00000100:00000001:0.0:1775768080.050134:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768080.050140:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775768080.050140:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768080.050143:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768080.050147:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1775768080.050149:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775768080.050152:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775768080.050153:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768080.050173:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768080.050176:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768080.050177:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1775768080.050178:0:36:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768080.050178:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1775768080.050183:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00000040:1.0:1775768080.050184:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fc700 x1862022294271488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768080.050188:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5332 00000400:00000200:0.0:1775768080.050197:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768080.050197:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768080.050199:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768080.050202:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000200:0.0:1775768080.050204:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 02000000:00000001:1.0:1775768080.050205:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:0.0:1775768080.050208:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 02000000:00000001:1.0:1775768080.050209:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768080.050216:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00100000:1.0:1775768080.050216:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294271488:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768080.050221:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775768080.050222:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768080.050225:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:0.0:1775768080.050227:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000001:1.0:1775768080.050228:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768080.050231:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 02000000:00000001:1.0:1775768080.050232:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:0.0:1775768080.050235:0:36:0:(genops.c:811:class_conn2export()) Process entered 02000000:00000010:1.0:1775768080.050236:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543e800. 00000020:00000040:0.0:1775768080.050237:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:0.0:1775768080.050240:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000001:1.0:1775768080.050241:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1775768080.050242:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:0.0:1775768080.050245:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000400:00000200:1.0:1775768080.050246:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:0.0:1775768080.050248:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000400:00000010:1.0:1775768080.050249:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000100:00000040:0.0:1775768080.050253:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000010:1.0:1775768080.050254:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000001:0.0:1775768080.050257:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:1.0:1775768080.050259:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294271488, portal 10 00000100:00000040:0.0:1775768080.050261:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768080.050262:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768080.050265:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641198336 : -112966068353280 : ffff9942099fc700) 00000100:00000001:0.0:1775768080.050266:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768080.050272:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:0.0:1775768080.050274:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768080.050276:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768080.050279:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:1.0:1775768080.050279:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fc700 x1862022294271488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768105 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1775768080.050283:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768080.050287:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768080.050291:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775768080.050293:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:0.0:1775768080.050297:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768080.050297:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1775768080.050300:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768080.050304:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000100:00000010:0.0:1775768080.050306:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ff800. 00000400:00000010:1.0:1775768080.050308:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000020:00000040:0.0:1775768080.050309:0:36:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:0.0:1775768080.050314:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000200:1.0:1775768080.050315:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294271488, offset 0 02000000:00000001:0.0:1775768080.050316:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768080.050319:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768080.050321:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768080.050323:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768080.050327:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 02000000:00000010:0.0:1775768080.050331:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220029800. 00000400:00000200:1.0:1775768080.050334:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768080.050336:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768080.050340:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00100000:0.0:1775768080.050344:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ff800 x1862022294271872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775768080.050345:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768080.050351:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768080.050356:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1775768080.050361:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1775768080.050361:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00000001:0.0:1775768080.050365:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768080.050365:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225294:225294:256:4294967295] 192.168.202.156@tcp LPNI seq info [225290:225290:8:4294967295] 00000100:00000040:0.0:1775768080.050371:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000aa4e8b71] to pc [ptlrpcd_rcv+-1] req@ffff9942099ff800 x1862022294271872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768080.050377:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768080.050384:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5342) 00000400:00000200:1.0:1775768080.050384:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768080.050393:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768080.050397:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768080.050401:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000800:00000200:1.0:1775768080.050405:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768080.050407:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775768080.050411:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768080.050412:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1775768080.050415:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768080.050416:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768080.050419:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768080.050421:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768080.050424:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768080.050425:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768080.050426:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768080.050429:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fc700 x1862022294271488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768105 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768080.050435:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768080.050437:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fc700 x1862022294271488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768105 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768080.050442:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768080.050443:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768080.050445:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768080.050448:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:1.0:1775768080.050450:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768080.050451:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300004883 00000100:00000001:1.0:1775768080.050453:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300006205 00000100:00000001:1.0:1775768080.050455:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300004883 00000100:00000001:1.0:1775768080.050457:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300006205 00000100:00000001:1.0:1775768080.050458:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768080.050459:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768080.050460:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768080.050463:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768080/real 1775768080] req@ffff9942099fc700 x1862022294271488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768080.050468:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768080.050469:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768080.050470:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768080.050472:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768080.050473:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768080.050474:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768080.050476:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fc700 x1862022294271488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768080.050482:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fc700 x1862022294271488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768080.050486:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768080.050487:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768080.050488:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768080.050489:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768080.050491:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fc700 x1862022294271488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768080.050495:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768080.050497:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fc700 x1862022294271488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768105 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768080.050501:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768080.050503:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768080.050503:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768080.050504:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768080.050505:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768080.050507:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768080.050509:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768080.050511:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768080.050512:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768080.050515:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fc700 x1862022294271488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768105 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768080.050520:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294271488:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768080.050522:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768080.050524:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ff100 x1862022294271616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768080.050533:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768080.050534:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768080.050535:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768080.050536:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768080.050538:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768080.050540:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294271616:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768080.050541:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768080.050543:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768080.050544:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768080.050545:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768080.050547:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543d800. 02000000:00000001:1.0:1775768080.050548:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768080.050550:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768080.050551:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000400:00000010:1.0:1775768080.050553:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000200:1.0:1775768080.050555:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294271616, portal 10 00000100:00000001:1.0:1775768080.050556:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768080.050557:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641209088 : -112966068342528 : ffff9942099ff100) 00000100:00000040:1.0:1775768080.050561:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ff100 x1862022294271616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768105 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768080.050565:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768080.050566:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768080.050568:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768080.050569:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294271616, offset 0 00000400:00000200:1.0:1775768080.050571:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768080.050574:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768080.050576:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768080.050577:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768080.050579:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768080.050580:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768080.050582:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768080.050585:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225295:225295:256:4294967295] 192.168.202.156@tcp LPNI seq info [225291:225291:8:4294967295] 00000400:00000200:1.0:1775768080.050587:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768080.050592:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768080.050594:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768080.050596:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000800:00000200:1.0:1775768080.050599:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768080.050601:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775768080.050603:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768080.050605:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1775768080.050608:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768080.050609:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768080.050611:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768080.050613:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768080.050615:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768080.050616:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768080.050617:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768080.050619:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ff100 x1862022294271616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768105 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768080.050624:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768080.050626:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ff100 x1862022294271616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768105 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768080.050630:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768080.050631:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768080.050633:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768080.050634:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:1.0:1775768080.050636:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768080.050637:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300004875 00000100:00000001:1.0:1775768080.050639:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300006205 00000100:00000001:1.0:1775768080.050640:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300004875 00000100:00000001:1.0:1775768080.050641:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300006205 00000100:00000001:1.0:1775768080.050643:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768080.050644:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768080.050646:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768080.050648:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768080/real 1775768080] req@ffff9942099ff100 x1862022294271616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768080.050653:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768080.050654:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768080.050654:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768080.050656:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768080.050657:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768080.050658:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768080.050660:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ff100 x1862022294271616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768080.050665:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ff100 x1862022294271616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768080.050669:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768080.050670:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768080.050671:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768080.050672:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768080.050673:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ff100 x1862022294271616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768080.050678:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768080.050681:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ff100 x1862022294271616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768105 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768080.050688:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768080.050690:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768080.050691:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768080.050692:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768080.050694:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768080.050696:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768080.050698:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768080.050702:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768080.050704:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768080.050708:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ff100 x1862022294271616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768105 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768080.050717:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294271616:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768080.050721:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768080.050725:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fd500 x1862022294271744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768080.050732:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768080.050733:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768080.050736:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768080.050738:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768080.050740:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768080.050743:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294271744:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768080.050747:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768080.050748:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768080.050750:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768080.050752:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768080.050755:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543f800. 02000000:00000001:1.0:1775768080.050757:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768080.050759:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768080.050761:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000400:00000010:1.0:1775768080.050763:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000200:1.0:1775768080.050766:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294271744, portal 4 00000100:00000001:1.0:1775768080.050767:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768080.050768:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641201920 : -112966068349696 : ffff9942099fd500) 00000100:00000040:1.0:1775768080.050772:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fd500 x1862022294271744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768105 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768080.050776:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768080.050778:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768080.050780:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768080.050781:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294271744, offset 0 00000400:00000200:1.0:1775768080.050784:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768080.050787:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768080.050789:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768080.050791:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768080.050793:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768080.050795:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768080.050797:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768080.050799:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225296:225296:256:4294967295] 192.168.202.156@tcp LPNI seq info [225292:225292:8:4294967295] 00000400:00000200:1.0:1775768080.050802:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768080.050808:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768080.050812:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768080.050816:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000800:00000200:1.0:1775768080.050820:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768080.050822:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775768080.050827:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768080.050829:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1775768080.050833:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768080.050835:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768080.050838:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768080.050841:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768080.050844:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768080.050846:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768080.050847:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768080.050850:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fd500 x1862022294271744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768105 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768080.050856:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768080.050857:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fd500 x1862022294271744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768105 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768080.050862:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768080.050863:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768080.050866:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768080.050868:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:1.0:1775768080.050870:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768080.050872:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300004898 00000100:00000001:1.0:1775768080.050873:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300006205 00000100:00000001:1.0:1775768080.050875:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300004898 00000100:00000001:1.0:1775768080.050876:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300006205 00000100:00000001:1.0:1775768080.050877:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768080.050879:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768080.050880:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768080.050882:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768080/real 1775768080] req@ffff9942099fd500 x1862022294271744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768080.050887:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768080.050888:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768080.050889:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768080.050890:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768080.050891:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768080.050892:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768080.050894:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fd500 x1862022294271744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768080.050899:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fd500 x1862022294271744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768080.050903:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768080.050904:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768080.050905:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768080.050906:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768080.050908:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fd500 x1862022294271744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768080.050912:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768080.050914:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fd500 x1862022294271744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768105 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768080.050919:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768080.050921:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768080.050922:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768080.050923:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768080.050924:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768080.050927:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768080.050928:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768080.050930:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768080.050931:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768080.050934:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fd500 x1862022294271744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768105 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768080.050939:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294271744:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768080.050941:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768080.050942:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768080.050944:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fc700 x1862022294271488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768105 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768080.050948:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768080.050949:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768080.050950:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543e800. 02000000:00000001:1.0:1775768080.050952:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768080.050953:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768080.050953:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:1.0:1775768080.050955:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768080.050956:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029400. 02000000:00000001:1.0:1775768080.050957:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768080.050958:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768080.050959:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fc700. 00000100:00000001:1.0:1775768080.050961:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768080.050962:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768080.050963:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768080.050965:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ff100 x1862022294271616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768105 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768080.050969:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768080.050970:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768080.050971:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543d800. 02000000:00000001:1.0:1775768080.050972:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768080.050973:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768080.050973:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:1.0:1775768080.050974:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768080.050975:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002bc00. 02000000:00000001:1.0:1775768080.050976:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768080.050977:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768080.050978:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ff100. 00000100:00000001:1.0:1775768080.050979:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768080.050980:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768080.050981:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768080.050982:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fd500 x1862022294271744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768105 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768080.050986:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768080.050987:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768080.050988:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543f800. 02000000:00000001:1.0:1775768080.050989:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768080.050989:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768080.050990:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:1.0:1775768080.050991:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768080.050992:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002a800. 02000000:00000001:1.0:1775768080.050993:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768080.050994:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768080.050995:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fd500. 00000100:00000001:1.0:1775768080.050996:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768080.050997:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768080.050998:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768080.051000:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768080.051001:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768080.051002:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768080.051003:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768080.051004:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768080.051007:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ff800 x1862022294271872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768080.051011:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768080.051012:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768080.051013:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768080.051014:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768080.051015:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768080.051017:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294271872:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768080.051019:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768080.051020:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768080.051021:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768080.051022:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768080.051024:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543f800. 02000000:00000001:1.0:1775768080.051026:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768080.051028:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768080.051029:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000400:00000010:1.0:1775768080.051031:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000200:1.0:1775768080.051033:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294271872, portal 4 00000100:00000001:1.0:1775768080.051034:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768080.051035:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641210880 : -112966068340736 : ffff9942099ff800) 00000100:00000040:1.0:1775768080.051039:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ff800 x1862022294271872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768105 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768080.051044:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768080.051045:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768080.051046:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768080.051048:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294271872, offset 0 00000400:00000200:1.0:1775768080.051050:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768080.051052:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768080.051054:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768080.051056:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768080.051058:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768080.051060:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768080.051061:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768080.051063:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225297:225297:256:4294967295] 192.168.202.156@tcp LPNI seq info [225293:225293:8:4294967295] 00000400:00000200:1.0:1775768080.051066:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768080.051070:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768080.051073:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768080.051076:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000800:00000200:1.0:1775768080.051080:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768080.051082:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775768080.051086:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768080.051089:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1775768080.051093:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768080.051095:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768080.051097:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768080.051101:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768080.051104:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768080.051105:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768080.051108:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768080.051111:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ff800 x1862022294271872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768105 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768080.051132:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768080.051135:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ff800 x1862022294271872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768105 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768080.051143:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768080.051144:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768080.051148:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768080.051151:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:1.0:1775768080.051154:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768080.051172:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300004902 00000100:00000001:1.0:1775768080.051173:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300006205 00000100:00000001:1.0:1775768080.051175:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300004902 00000100:00000001:1.0:1775768080.051178:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300006205 00000100:00000001:1.0:1775768080.051179:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768080.051181:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768080.051182:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768080.051185:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768080/real 1775768080] req@ffff9942099ff800 x1862022294271872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768080.051190:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768080.051191:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768080.051192:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768080.051194:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768080.051195:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768080.051197:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768080.051199:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ff800 x1862022294271872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768080.051204:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ff800 x1862022294271872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768080.051208:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768080.051209:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768080.051210:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768080.051211:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768080.051212:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ff800 x1862022294271872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768080.051217:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768080.051219:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ff800 x1862022294271872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768105 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768080.051223:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768080.051225:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768080.051226:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768080.051226:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768080.051228:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768080.051231:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768080.051232:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768080.051235:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768080.051236:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768080.051239:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ff800 x1862022294271872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768105 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768080.051244:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294271872:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768080.051246:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768080.051248:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768080.051249:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ff800 x1862022294271872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768105 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768080.051253:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768080.051254:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768080.051255:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543f800. 02000000:00000001:1.0:1775768080.051256:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768080.051257:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768080.051258:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:1.0:1775768080.051259:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768080.051260:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029800. 02000000:00000001:1.0:1775768080.051261:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768080.051262:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768080.051263:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ff800. 00000100:00000001:1.0:1775768080.051264:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768080.051265:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768080.051267:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768080.051270:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768080.051271:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768080.051273:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768080.051275:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768080.113255:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768080.113260:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768080.113265:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768080.113267:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768080.113269:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768080.113270:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768080.113272:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768080.113273:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768080.113275:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768080.113276:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768080.113291:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768080.113293:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768080.113296:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768080.177280:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768080.177286:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768080.177288:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768080.177294:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768080.177298:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768080.177302:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768080.177305:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768080.177307:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768080.177310:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768081.073234:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768081.073234:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768081.073235:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768081.073242:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768081.073242:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768081.073242:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768081.073246:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768081.073246:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768081.073247:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768081.073254:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775768081.073254:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768081.073259:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768081.073260:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768081.073263:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768081.073263:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768081.073264:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768081.073267:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768081.073269:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768081.073271:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768081.073273:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768081.073274:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768081.073283:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768081.073286:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768081.073290:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768081.073290:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768081.073294:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768081.073297:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768081.137261:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768081.137266:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768081.137271:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768081.137272:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768081.137275:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768081.137276:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768081.137278:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768081.137278:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768081.137281:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768081.137282:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768081.137345:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768081.137349:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768081.137352:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768081.201230:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768081.201235:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768081.201237:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768081.201242:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768081.201245:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768081.201248:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768081.201250:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768081.201252:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768081.201254:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768081.393257:0:14953:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4300006205, at 4300006208 00000100:00000040:1.0:1775768081.393257:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4300006205, at 4300006208 00000100:00000010:0.0:1775768081.393272:0:14953:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cbaf60. 00000100:00000010:1.0:1775768081.393272:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cba180. 00000100:00000001:3.0:1775768082.097206:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768082.097213:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768082.097216:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768082.097217:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768082.097219:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768082.097222:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:3.0:1775768082.097222:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768082.097223:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768082.097223:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768082.097225:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768082.097229:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:2.0:1775768082.097229:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768082.097229:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768082.097233:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768082.097233:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768082.097234:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768082.097237:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768082.097237:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768082.097238:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768082.097242:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768082.097244:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768082.097248:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768082.097251:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768082.097254:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768082.097255:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768082.097257:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768082.097258:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768082.161233:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768082.161239:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768082.161241:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768082.161243:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768082.161247:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768082.161249:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768082.161250:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768082.161252:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768082.161254:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768082.161255:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768082.161258:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768082.161258:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768082.161260:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768082.161261:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768082.161263:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768082.161264:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768082.161278:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768082.161280:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768082.161283:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768082.225261:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768082.225266:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768082.225268:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768082.225272:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768082.225275:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768082.225278:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768082.225280:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768082.225282:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768082.225285:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768083.121243:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768083.121246:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768083.121253:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768083.121253:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768083.121256:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768083.121258:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768083.121265:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:2.0:1775768083.121266:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775768083.121266:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768083.121273:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768083.121275:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768083.121277:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768083.121278:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768083.121283:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768083.121284:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768083.121285:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768083.121288:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768083.121290:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768083.121293:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768083.121296:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768083.121299:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768083.121303:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768083.121309:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768083.121312:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768083.121316:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768083.121323:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768083.121327:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768083.185295:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768083.185304:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768083.185312:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768083.185314:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768083.185319:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768083.185320:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768083.185324:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768083.185326:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768083.185329:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768083.185330:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768083.185390:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768083.185394:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768083.185398:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768083.249274:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768083.249283:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768083.249285:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768083.249290:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768083.249295:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768083.249299:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768083.249301:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768083.249303:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768083.249306:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768084.145207:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768084.145215:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768084.145217:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768084.145223:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:3.0:1775768084.145224:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775768084.145227:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768084.145230:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768084.145231:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:2.0:1775768084.145233:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768084.145236:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768084.145237:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768084.145241:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768084.145241:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768084.145246:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768084.145248:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768084.145251:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768084.145255:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768084.145259:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768084.145265:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768084.145272:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768084.145276:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768084.145282:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768084.145296:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768084.145299:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768084.145302:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768084.145304:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768084.145307:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768084.209300:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768084.209306:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768084.209310:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768084.209311:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768084.209314:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768084.209316:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768084.209318:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768084.209318:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768084.209321:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768084.209321:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768084.209364:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768084.209370:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768084.209374:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768084.273233:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768084.273241:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768084.273243:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768084.273249:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768084.273254:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768084.273258:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768084.273261:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768084.273263:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768084.273266:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768085.169207:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768085.169213:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768085.169213:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768085.169217:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768085.169218:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768085.169221:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768085.169223:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:2.0:1775768085.169225:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768085.169227:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768085.169231:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768085.169232:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768085.169236:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768085.169236:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768085.169239:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768085.169242:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768085.169243:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1775768085.169247:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775768085.169247:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768085.169252:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768085.169254:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775768085.169258:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768085.169261:0:36:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768085.169265:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768085.169272:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5337 00000400:00000200:0.0:1775768085.169281:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000080:00000001:3.0:1775768085.169281:0:13471:0:(rw.c:1481:ll_writepages()) Process entered 00000080:00000001:3.0:1775768085.169285:0:13471:0:(file.c:5250:cl_sync_file_range()) Process entered 00000400:00000200:0.0:1775768085.169288:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000020:00000001:3.0:1775768085.169288:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00000001:2.0:1775768085.169289:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:0.0:1775768085.169294:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000020:00000010:3.0:1775768085.169294:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff99420693e000. 00000100:00000001:2.0:1775768085.169295:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768085.169298:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768085.169301:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00100000:2.0:1775768085.169301:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000020:00000010:3.0:1775768085.169303:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dce600. 00000100:00000001:0.0:1775768085.169306:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775768085.169307:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000010:3.0:1775768085.169310:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210dcea00. 00000100:00000001:2.0:1775768085.169311:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768085.169313:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768085.169315:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000010:3.0:1775768085.169316:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207981de0. 00000100:00000001:2.0:1775768085.169317:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768085.169321:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768085.169323:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000008:00000010:3.0:1775768085.169324:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994208b50000. 00000020:00000001:0.0:1775768085.169330:0:36:0:(genops.c:811:class_conn2export()) Process entered 00020000:00000010:3.0:1775768085.169330:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994218000300. 00000020:00000040:0.0:1775768085.169334:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00008000:00000010:3.0:1775768085.169336:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994207e08e20. 00000020:00000001:0.0:1775768085.169338:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768085.169342:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000080:00000010:3.0:1775768085.169343:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99420541b1f8. 00000020:00000001:0.0:1775768085.169347:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000080:00000010:3.0:1775768085.169349:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077c0248. 00000020:00000001:0.0:1775768085.169352:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000010:3.0:1775768085.169353:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dce200. 00000008:00000010:3.0:1775768085.169359:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942086f8420. 00000100:00000040:0.0:1775768085.169360:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768085.169365:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00020000:00000010:3.0:1775768085.169365:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994207e8e400. 00000100:00000040:0.0:1775768085.169370:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00008000:00000010:3.0:1775768085.169371:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994209270a90. 00000100:00000001:0.0:1775768085.169376:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000080:00000010:3.0:1775768085.169378:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942090e33c0. 00000020:00000040:0.0:1775768085.169382:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000020:00000001:3.0:1775768085.169382:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107590086664 : -112966119464952 : ffff99420693e008) 00000020:00001000:3.0:1775768085.169387:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff99420693e000 00000100:00080000:0.0:1775768085.169390:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000020:00000001:3.0:1775768085.169392:0:13471:0:(cl_object.c:151:cl_object_top()) ffff99420791a5f8 -> ffff99420791a5f8 00000100:00000001:0.0:1775768085.169394:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768085.169396:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 02000000:00000001:0.0:1775768085.169399:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000080:00000001:3.0:1775768085.169400:0:13471:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:3.0:1775768085.169404:0:13471:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x10176:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 02000000:00000001:0.0:1775768085.169405:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1775768085.169409:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 02000000:00000001:0.0:1775768085.169410:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768085.169411:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:3.0:1775768085.169414:0:13471:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768085.169416:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00020000:00000002:3.0:1775768085.169418:0:13471:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x10176:0x0]io ffff9942077c02a8 type 6 ignore/verify layout 1/0 00000100:00000001:0.0:1775768085.169420:0:36:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:3.0:1775768085.169423:0:13471:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207dfb6c8) owner 0000000000000000/ffff994220080000 00020000:00000001:3.0:1775768085.169427:0:13471:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:3.0:1775768085.169429:0:13471:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:3.0:1775768085.169433:0:13471:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00020000:00000001:3.0:1775768085.169434:0:13471:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768085.169436:0:13471:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000010:0.0:1775768085.169438:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fc000. 00020000:00000001:3.0:1775768085.169438:0:13471:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:3.0:1775768085.169442:0:13471:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1775768085.169443:0:36:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 00000020:00000001:3.0:1775768085.169444:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:3.0:1775768085.169447:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:3.0:1775768085.169449:0:13471:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 02000000:00000001:0.0:1775768085.169450:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00020000:00000002:3.0:1775768085.169450:0:13471:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207dfb6c8) owner 0000000000000000/ffff994220080000 02000000:00000001:0.0:1775768085.169452:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768085.169453:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768085.169456:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000020:00000001:3.0:1775768085.169456:0:13471:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:3.0:1775768085.169459:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 02000000:00000001:0.0:1775768085.169460:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00020000:00000001:3.0:1775768085.169461:0:13471:0:(lov_io.c:836:lov_io_iter_init()) Process entered 02000000:00000001:0.0:1775768085.169464:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00020000:00200000:3.0:1775768085.169464:0:13471:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00000002:3.0:1775768085.169470:0:13471:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:3.0:1775768085.169473:0:13471:0:(lov_io.c:155:lov_sub_get()) Process entered 02000000:00000010:0.0:1775768085.169474:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002b800. 00020000:00000001:3.0:1775768085.169474:0:13471:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000020:00000001:3.0:1775768085.169477:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:3.0:1775768085.169480:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff99420693ecf0. 00000100:00000001:0.0:1775768085.169481:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1775768085.169483:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dce000. 00000020:00000010:3.0:1775768085.169486:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210dcf000. 00000020:00000010:3.0:1775768085.169488:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942079817e0. 00000100:00100000:0.0:1775768085.169491:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fc000 x1862022294272000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000008:00000010:3.0:1775768085.169492:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994208b51650. 00020000:00000010:3.0:1775768085.169495:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994218000000. 00008000:00000010:3.0:1775768085.169500:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994207e0ade8. 00000100:00000001:0.0:1775768085.169509:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768085.169513:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768085.169519:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008bcb9792] to pc [ptlrpcd_rcv+-1] req@ffff9942099fc000 x1862022294272000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768085.169544:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768085.169547:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768085.169552:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775768085.169553:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768085.169555:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768085.169561:0:36:0:(import.c:484:import_select_connection()) Process entered 00000080:00000010:3.0:1775768085.169562:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99420541b0a8. 00000100:00080000:0.0:1775768085.169565:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5337 00000080:00000010:3.0:1775768085.169568:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077c1da8. 00000100:00000001:1.0:1775768085.169569:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:0.0:1775768085.169573:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768085.169574:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000010:3.0:1775768085.169576:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcf400. 00000400:00000200:0.0:1775768085.169577:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:1.0:1775768085.169577:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00080000:0.0:1775768085.169582:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000008:00000010:3.0:1775768085.169585:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942086f8160. 00000100:00000040:1.0:1775768085.169586:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fc000 x1862022294272000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775768085.169589:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00020000:00000010:3.0:1775768085.169592:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994207e8ed00. 00000100:00000001:0.0:1775768085.169594:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768085.169599:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768085.169602:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768085.169603:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:1.0:1775768085.169604:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768085.169606:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768085.169608:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 02000000:00000001:1.0:1775768085.169608:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:0.0:1775768085.169611:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000001:1.0:1775768085.169612:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:0.0:1775768085.169613:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 02000000:00000001:1.0:1775768085.169616:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768085.169617:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:0.0:1775768085.169621:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00100000:1.0:1775768085.169623:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294272000:192.168.202.156@tcp:250:kworker.0 00000100:00000040:0.0:1775768085.169626:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768085.169628:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1775768085.169630:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1775768085.169633:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:0.0:1775768085.169634:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768085.169637:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 02000000:00000001:1.0:1775768085.169637:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1775768085.169641:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 02000000:00000001:1.0:1775768085.169642:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:0.0:1775768085.169645:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768085.169647:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775768085.169647:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543e800. 02000000:00000001:0.0:1775768085.169649:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775768085.169651:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768085.169655:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768085.169658:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768085.169659:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768085.169662:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000010:1.0:1775768085.169664:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000008:00100000:0.0:1775768085.169666:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775768085.169669:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775768085.169670:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000001:0.0:1775768085.169671:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768085.169677:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ff800. 00000100:00000200:1.0:1775768085.169679:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294272000, portal 25 00000020:00000040:0.0:1775768085.169680:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:0.0:1775768085.169685:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1775768085.169685:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768085.169687:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641196544 : -112966068355072 : ffff9942099fc000) 02000000:00000001:0.0:1775768085.169688:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768085.169693:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768085.169694:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:1.0:1775768085.169697:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fc000 x1862022294272000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768110 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1775768085.169698:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768085.169705:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220028400. 00000100:00000001:1.0:1775768085.169706:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1775768085.169711:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768085.169711:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000100:00100000:0.0:1775768085.169719:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ff800 x1862022294272128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00008000:00000010:3.0:1775768085.169720:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994209270ef0. 00000400:00000010:1.0:1775768085.169722:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000080:00000010:3.0:1775768085.169726:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942090e3960. 00000100:00000200:1.0:1775768085.169729:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294272000, offset 0 00000020:00000001:3.0:1775768085.169732:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107590089976 : -112966119461640 : ffff99420693ecf8) 00000100:00000001:0.0:1775768085.169735:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000020:00001000:3.0:1775768085.169738:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff99420693ecf0 00000100:00000001:0.0:1775768085.169739:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768085.169742:0:13471:0:(cl_object.c:151:cl_object_top()) ffff99420791ae90 -> ffff99420791a5f8 00000400:00000200:1.0:1775768085.169745:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00000040:0.0:1775768085.169746:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000aa4e8b71] to pc [ptlrpcd_rcv+-1] req@ffff9942099ff800 x1862022294272128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775768085.169746:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:3.0:1775768085.169751:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:3.0:1775768085.169754:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:3.0:1775768085.169758:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768085.169759:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:3.0:1775768085.169764:0:13471:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768085.169768:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768085.169769:0:13471:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107612435560 : -112966097116056 : ffff994207e8e468) 00000400:00000200:1.0:1775768085.169771:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00080000:0.0:1775768085.169774:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000001:3.0:1775768085.169776:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000400:00000200:1.0:1775768085.169780:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:0.0:1775768085.169783:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775768085.169786:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768085.169788:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768085.169792:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768085.169797:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5337 00000400:00000200:1.0:1775768085.169798:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768085.169806:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768085.169809:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768085.169814:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225298:225298:256:4294967295] 192.168.202.156@tcp LPNI seq info [225294:225294:8:4294967295] 00000400:00000200:1.0:1775768085.169820:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768085.169831:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000020:00000001:3.0:1775768085.169839:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768085.169840:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000800:00000200:1.0:1775768085.169841:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00020000:00000001:3.0:1775768085.169845:0:13471:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768085.169847:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000800:00000010:1.0:1775768085.169849:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000020:00000001:3.0:1775768085.169849:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768085.169851:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000020:00000001:3.0:1775768085.169853:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00020000:00000001:3.0:1775768085.169855:0:13471:0:(lov_io.c:1108:lov_io_lock()) Process entered 00000100:00000040:0.0:1775768085.169858:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00020000:00000001:3.0:1775768085.169858:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775768085.169860:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00000800:00000200:1.0:1775768085.169863:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:0.0:1775768085.169864:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:3.0:1775768085.169864:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:3.0:1775768085.169867:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768085.169871:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000800:00000200:1.0:1775768085.169871:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000020:00000001:3.0:1775768085.169871:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768085.169875:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768085.169878:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00020000:00000001:3.0:1775768085.169879:0:13471:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768085.169883:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:0.0:1775768085.169885:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000001:3.0:1775768085.169885:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775768085.169886:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000040:0.0:1775768085.169889:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:3.0:1775768085.169889:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768085.169894:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000001:3.0:1775768085.169894:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00000800:00000200:1.1:1775768085.169894:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00020000:00000001:3.0:1775768085.169897:0:13471:0:(lov_io.c:1114:lov_io_start()) Process entered 00000020:00000040:0.0:1775768085.169898:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00020000:00000001:3.0:1775768085.169900:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775768085.169903:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00000020:00000001:0.0:1775768085.169905:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000800:00000100:1.0:1775768085.169906:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000008:00000001:3.0:1775768085.169906:0:13471:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000008:00000001:3.0:1775768085.169910:0:13471:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000800:00000010:1.0:1775768085.169911:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000020:00000001:0.0:1775768085.169912:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000008:00000001:3.0:1775768085.169916:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000100:00000040:0.0:1775768085.169919:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000200:1.0:1775768085.169919:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000008:00000020:3.0:1775768085.169919:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:3.0:1775768085.169923:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768085.169925:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000008:00000001:3.0:1775768085.169927:0:13471:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000100:00000040:0.0:1775768085.169930:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000200:1.0:1775768085.169931:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000100:00000001:0.0:1775768085.169935:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000008:00000001:3.0:1775768085.169936:0:13471:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000020:00000040:0.0:1775768085.169937:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000400:00000200:1.0:1775768085.169938:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000008:00000001:3.0:1775768085.169939:0:13471:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107754940544 : -112965954611072 : ffff994210675880) 00000400:00000010:1.0:1775768085.169942:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000008:00000020:3.0:1775768085.169948:0:13471:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994210675880 ready 0|-|- wr 1|-|+ rd 0|- 10 in flight 00000100:00000001:1.0:1775768085.169949:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:0.0:1775768085.169952:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000008:00000001:3.0:1775768085.169954:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:3.0:1775768085.169955:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:3.0:1775768085.169957:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:1.0:1775768085.169960:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fc000 x1862022294272000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768110 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768085.169961:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1775768085.169961:0:13471:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000008:00000020:3.0:1775768085.169965:0:13471:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994218232fc0 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 02000000:00000001:0.0:1775768085.169966:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768085.169970:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768085.169975:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1775768085.169975:0:13471:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768085.169982:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00000001:3.0:1775768085.169982:0:13471:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768085.169988:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000020:00000001:3.0:1775768085.169988:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768085.169991:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1775768085.169993:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00020000:00000001:3.0:1775768085.169997:0:13471:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1775768085.169998:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768085.170001:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768085.170002:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768085.170005:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00000100:00000040:1.0:1775768085.170006:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fc000 x1862022294272000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768110 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1775768085.170010:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fd500. 00020000:00000001:3.0:1775768085.170010:0:13471:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00020000:00000001:3.0:1775768085.170014:0:13471:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000020:00000040:0.0:1775768085.170017:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:3.0:1775768085.170017:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000001:3.0:1775768085.170021:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 02000000:00000001:0.0:1775768085.170024:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00020000:00000001:3.0:1775768085.170025:0:13471:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768085.170027:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768085.170031:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768085.170032:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00020000:00000001:3.0:1775768085.170032:0:13471:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 02000000:00000001:0.0:1775768085.170036:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1775768085.170036:0:2399:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:0.0:1775768085.170041:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768085.170043:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00000100:00000001:1.0:1775768085.170045:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768085.170046:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 02000000:00000010:0.0:1775768085.170047:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220029800. 00020000:00000001:3.0:1775768085.170051:0:13471:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00000100:00000001:0.0:1775768085.170053:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768085.170053:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:3.0:1775768085.170054:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000001:1.0:1775768085.170058:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4300011325 00000020:00000001:3.0:1775768085.170058:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000020:00000001:3.0:1775768085.170061:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000100:00100000:0.0:1775768085.170063:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fd500 x1862022294272256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00020000:00000001:3.0:1775768085.170065:0:13471:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775768085.170066:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cbaae0. 00000100:00000001:1.0:1775768085.170085:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775768085.170087:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00020000:00000001:3.0:1775768085.170087:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768085.170091:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4300011325 00000100:00000001:0.0:1775768085.170092:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768085.170092:0:13471:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000020:00000001:3.0:1775768085.170096:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000100:00000010:1.0:1775768085.170097:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba360. 00000100:00000040:0.0:1775768085.170099:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f246bb2b] to pc [ptlrpcd_rcv+-1] req@ffff9942099fd500 x1862022294272256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775768085.170101:0:13471:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000100:00000001:1.0:1775768085.170102:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000020:00000001:3.0:1775768085.170105:0:13471:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000100:00000001:1.0:1775768085.170107:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1775768085.170109:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000100:00000001:1.0:1775768085.170112:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768085.170114:0:13471:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00000100:00000001:1.0:1775768085.170117:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00020000:00000001:3.0:1775768085.170118:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775768085.170121:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000100:00000001:0.0:1775768085.170124:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768085.170125:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768085/real 1775768085] req@ffff9942099fc000 x1862022294272000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775768085.170125:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000100:00080000:0.0:1775768085.170129:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00020000:00000001:3.0:1775768085.170129:0:13471:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768085.170135:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768085.170140:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00020000:00000001:3.0:1775768085.170142:0:13471:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000100:00000001:0.0:1775768085.170145:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000020:00000001:3.0:1775768085.170146:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000400:00000001:1.0:1775768085.170148:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:0.0:1775768085.170149:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000020:00000001:3.0:1775768085.170150:0:13471:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1775768085.170151:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768085.170172:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768085.170176:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000020:00000001:3.0:1775768085.170177:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00000400:00000010:1.0:1775768085.170178:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00020000:00000001:3.0:1775768085.170181:0:13471:0:(lov_io.c:675:lov_io_fini()) Process entered 00000100:00000001:0.0:1775768085.170183:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775768085.170184:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00020000:00000001:3.0:1775768085.170184:0:13471:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:3.0:1775768085.170188:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00000100:00000200:1.0:1775768085.170192:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fc000 x1862022294272000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775768085.170192:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000100:00080000:0.0:1775768085.170193:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5337 00000020:00001000:3.0:1775768085.170194:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff99420693ecf0 00000020:00000010:3.0:1775768085.170199:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210dcf000. 00000400:00000200:0.0:1775768085.170205:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000010:3.0:1775768085.170207:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942079817e0. 00000008:00000010:3.0:1775768085.170212:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994208b51650. 00000100:00000200:1.0:1775768085.170213:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fc000 x1862022294272000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000010:3.0:1775768085.170220:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994218000000. 00000400:00000200:0.0:1775768085.170221:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:1.0:1775768085.170225:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00080000:0.0:1775768085.170226:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00008000:00000010:3.0:1775768085.170227:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994207e0ade8. 00000100:00000001:1.0:1775768085.170229:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768085.170233:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768085.170234:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768085.170236:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775768085.170238:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768085.170240:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000080:00000010:3.0:1775768085.170240:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99420541b0a8. 00000100:00000001:0.0:1775768085.170243:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00100000:1.0:1775768085.170247:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fc000 x1862022294272000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775768085.170248:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000080:00000010:3.0:1775768085.170248:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077c1da8. 00000020:00000040:0.0:1775768085.170252:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000010:3.0:1775768085.170255:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dce000. 00000020:00000001:0.0:1775768085.170257:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000008:00000010:3.0:1775768085.170260:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942086f8160. 00000020:00000040:0.0:1775768085.170262:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00020000:00000010:3.0:1775768085.170265:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994207e8ed00. 00000020:00000001:0.0:1775768085.170268:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00008000:00000010:3.0:1775768085.170271:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994209270ef0. 00000020:00000001:0.0:1775768085.170275:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:1.0:1775768085.170275:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000080:00000010:3.0:1775768085.170276:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942090e3960. 00000020:00000010:3.0:1775768085.170283:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcf400. 00000100:00000040:0.0:1775768085.170286:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000010:3.0:1775768085.170287:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff99420693ecf0. 00000100:00000040:1.0:1775768085.170288:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fc000 x1862022294272000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768110 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768085.170293:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00020000:00000001:3.0:1775768085.170293:0:13471:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:3.0:1775768085.170297:0:13471:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000100:00000040:0.0:1775768085.170301:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000080:00000001:3.0:1775768085.170301:0:13471:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000100:00000001:1.0:1775768085.170302:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000080:00200000:3.0:1775768085.170305:0:13471:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x10176:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00000001:0.0:1775768085.170307:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:1.0:1775768085.170310:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:3.0:1775768085.170313:0:13471:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000040:0.0:1775768085.170314:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00000001:1.0:1775768085.170314:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:3.0:1775768085.170316:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000100:00000001:1.0:1775768085.170318:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1775768085.170320:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000020:00001000:3.0:1775768085.170320:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff99420693e000 00000100:00000001:1.0:1775768085.170324:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000020:00000010:3.0:1775768085.170324:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210dcea00. 00000100:00000001:0.0:1775768085.170325:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768085.170328:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000010:3.0:1775768085.170330:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207981de0. 02000000:00000001:0.0:1775768085.170331:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000008:00000010:3.0:1775768085.170335:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994208b50000. 02000000:00000001:0.0:1775768085.170337:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:1.0:1775768085.170337:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00020000:00000010:3.0:1775768085.170338:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994218000300. 02000000:00000001:0.0:1775768085.170344:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:3.0:1775768085.170344:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994207e08e20. 00000080:00000010:3.0:1775768085.170347:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99420541b1f8. 00000100:00000001:0.0:1775768085.170349:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00080000:1.0:1775768085.170349:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000080:00000010:3.0:1775768085.170351:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077c0248. 00000008:00100000:0.0:1775768085.170355:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000100:00000001:1.0:1775768085.170356:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000010:3.0:1775768085.170356:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dce600. 00000008:00000010:3.0:1775768085.170360:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942086f8420. 00000008:00000001:0.0:1775768085.170363:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:3.0:1775768085.170364:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994207e8e400. 00000100:00000040:1.0:1775768085.170368:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fc000 x1862022294272000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768110 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768085.170369:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:3.0:1775768085.170369:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994209270a90. 00000080:00000010:3.0:1775768085.170374:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942090e33c0. 00000100:00000010:0.0:1775768085.170377:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ff100. 00000020:00000010:3.0:1775768085.170379:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dce200. 00000020:00000040:0.0:1775768085.170384:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000010:3.0:1775768085.170384:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff99420693e000. 00000080:00000001:3.0:1775768085.170390:0:13471:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1775768085.170392:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00100000:1.0:1775768085.170393:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294272000:192.168.202.156@tcp:250:kworker.0 00000080:00000001:3.0:1775768085.170394:0:13471:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768085.170395:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768085.170401:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1775768085.170403:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1775768085.170404:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768085.170407:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768085.170407:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768085.170411:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fc000 x1862022294272000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768110 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1775768085.170412:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002a800. 00000100:00000001:0.0:1775768085.170418:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768085.170420:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768085.170423:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000100:00100000:0.0:1775768085.170426:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ff100 x1862022294272384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000010:1.0:1775768085.170427:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543e800. 02000000:00000001:1.0:1775768085.170433:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768085.170435:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:0.0:1775768085.170439:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000020:00000040:1.0:1775768085.170439:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:1.0:1775768085.170441:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1775768085.170442:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775768085.170445:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002b800. 00000100:00000040:0.0:1775768085.170448:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5e24158] to pc [ptlrpcd_rcv+-1] req@ffff9942099ff100 x1862022294272384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1775768085.170450:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768085.170453:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768085.170455:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fc000. 00000100:00000001:1.0:1775768085.170460:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768085.170462:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768085.170464:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768085.170465:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1775768085.170470:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775768085.170473:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:0.0:1775768085.170474:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775768085.170476:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768085.170477:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768085.170480:0:36:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768085.170481:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775768085.170483:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775768085.170484:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768085.170486:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00080000:0.0:1775768085.170487:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5337 00000100:00000040:1.0:1775768085.170491:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ff800 x1862022294272128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775768085.170493:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768085.170499:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:1.0:1775768085.170503:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1775768085.170504:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1775768085.170506:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768085.170511:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:0.0:1775768085.170512:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 02000000:00000001:1.0:1775768085.170515:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768085.170517:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1775768085.170519:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768085.170522:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768085.170527:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00100000:1.0:1775768085.170528:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294272128:192.168.202.156@tcp:38:kworker.0 00000020:00000001:0.0:1775768085.170532:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768085.170535:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000100:00000001:1.0:1775768085.170536:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000001:0.0:1775768085.170539:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000001:1.0:1775768085.170540:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:0.0:1775768085.170542:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 02000000:00000001:1.0:1775768085.170543:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768085.170547:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 02000000:00000001:1.0:1775768085.170547:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:0.0:1775768085.170552:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 02000000:00000010:1.0:1775768085.170553:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543e800. 02000000:00000001:1.0:1775768085.170558:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768085.170559:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768085.170562:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768085.170566:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000200:1.0:1775768085.170568:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768085.170569:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768085.170572:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000400:00000010:1.0:1775768085.170573:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000100:00080000:0.0:1775768085.170575:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768085.170579:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775768085.170581:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 02000000:00000001:0.0:1775768085.170582:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768085.170586:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000200:1.0:1775768085.170587:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294272128, portal 10 02000000:00000001:0.0:1775768085.170589:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768085.170592:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775768085.170592:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768085.170595:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641210880 : -112966068340736 : ffff9942099ff800) 00000008:00100000:0.0:1775768085.170596:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:0.0:1775768085.170601:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768085.170603:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768085.170607:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fc700. 00000100:00000040:1.0:1775768085.170608:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ff800 x1862022294272128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768110 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1775768085.170610:0:36:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:0.0:1775768085.170614:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768085.170616:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768085.170620:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768085.170622:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768085.170625:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768085.170625:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768085.170630:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 02000000:00000010:0.0:1775768085.170631:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002bc00. 00000400:00000010:1.0:1775768085.170633:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000001:0.0:1775768085.170635:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775768085.170637:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294272128, offset 0 00000100:00100000:0.0:1775768085.170642:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fc700 x1862022294272512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775768085.170642:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768085.170650:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768085.170657:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1775768085.170657:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00000001:0.0:1775768085.170662:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768085.170662:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:0.0:1775768085.170668:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003283b20a] to pc [ptlrpcd_rcv+-1] req@ffff9942099fc700 x1862022294272512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775768085.170669:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768085.170675:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768085.170681:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00000001:0.0:1775768085.170688:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768085.170688:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225299:225299:256:4294967295] 192.168.202.156@tcp LPNI seq info [225295:225295:8:4294967295] 00000100:00000040:0.0:1775768085.170693:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5347) 00000400:00000200:1.0:1775768085.170698:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768085.170707:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768085.170712:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768085.170717:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000800:00000200:1.0:1775768085.170721:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768085.170724:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775768085.170729:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768085.170732:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000100:1.0:1775768085.170737:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768085.170738:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768085.170742:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768085.170745:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768085.170749:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768085.170750:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768085.170753:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768085.170757:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ff800 x1862022294272128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768110 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768085.170766:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768085.170769:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ff800 x1862022294272128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768110 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768085.170777:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768085.170779:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768085.170782:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768085.170786:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:1.0:1775768085.170790:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768085.170793:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300010003 00000100:00000001:1.0:1775768085.170795:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300011325 00000100:00000001:1.0:1775768085.170797:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300010003 00000100:00000001:1.0:1775768085.170800:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300011325 00000100:00000001:1.0:1775768085.170802:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768085.170805:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768085.170807:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768085.170811:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768085/real 1775768085] req@ffff9942099ff800 x1862022294272128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768085.170820:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768085.170821:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768085.170822:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768085.170824:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768085.170825:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768085.170827:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768085.170829:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ff800 x1862022294272128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768085.170835:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ff800 x1862022294272128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768085.170839:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768085.170839:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768085.170841:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768085.170841:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768085.170843:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ff800 x1862022294272128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768085.170848:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768085.170850:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ff800 x1862022294272128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768110 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768085.170855:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768085.170856:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768085.170857:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768085.170858:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768085.170859:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768085.170862:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768085.170864:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768085.170866:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768085.170868:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768085.170870:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ff800 x1862022294272128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768110 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768085.170876:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294272128:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768085.170878:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768085.170880:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fd500 x1862022294272256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768085.170884:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768085.170885:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768085.170886:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768085.170888:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768085.170889:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768085.170891:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294272256:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768085.170893:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768085.170895:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768085.170896:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768085.170898:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768085.170900:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543fc00. 02000000:00000001:1.0:1775768085.170902:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768085.170904:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768085.170906:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000400:00000010:1.0:1775768085.170907:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000200:1.0:1775768085.170910:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294272256, portal 10 00000100:00000001:1.0:1775768085.170911:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768085.170912:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641201920 : -112966068349696 : ffff9942099fd500) 00000100:00000040:1.0:1775768085.170917:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fd500 x1862022294272256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768110 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768085.170925:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768085.170927:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768085.170929:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768085.170932:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294272256, offset 0 00000400:00000200:1.0:1775768085.170936:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768085.170940:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768085.170944:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768085.170948:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768085.170950:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768085.170953:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768085.170955:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768085.170958:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225300:225300:256:4294967295] 192.168.202.156@tcp LPNI seq info [225296:225296:8:4294967295] 00000400:00000200:1.0:1775768085.170962:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768085.170969:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768085.170974:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768085.170977:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000800:00000200:1.0:1775768085.170982:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768085.170984:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775768085.170988:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768085.170991:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000100:1.0:1775768085.170995:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768085.170997:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768085.171000:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768085.171003:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768085.171006:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768085.171008:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768085.171010:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768085.171013:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fd500 x1862022294272256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768110 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768085.171022:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768085.171025:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fd500 x1862022294272256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768110 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768085.171035:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768085.171036:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768085.171039:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768085.171041:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:1.0:1775768085.171043:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768085.171045:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300009995 00000100:00000001:1.0:1775768085.171047:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300011325 00000100:00000001:1.0:1775768085.171048:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300009995 00000100:00000001:1.0:1775768085.171049:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300011325 00000100:00000001:1.0:1775768085.171051:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768085.171052:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768085.171054:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768085.171056:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768085/real 1775768085] req@ffff9942099fd500 x1862022294272256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768085.171061:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768085.171062:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768085.171063:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768085.171064:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768085.171065:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768085.171066:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768085.171068:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fd500 x1862022294272256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768085.171090:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fd500 x1862022294272256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768085.171097:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768085.171099:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768085.171101:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768085.171102:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768085.171105:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fd500 x1862022294272256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768085.171113:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768085.171116:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fd500 x1862022294272256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768110 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768085.171124:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768085.171126:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768085.171128:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768085.171129:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768085.171131:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768085.171134:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768085.171136:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768085.171140:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768085.171142:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768085.171146:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fd500 x1862022294272256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768110 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768085.171172:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294272256:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768085.171177:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768085.171180:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ff100 x1862022294272384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768085.171186:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768085.171187:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768085.171189:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768085.171191:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768085.171193:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768085.171195:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294272384:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768085.171198:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768085.171199:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768085.171201:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768085.171203:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768085.171206:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543dc00. 02000000:00000001:1.0:1775768085.171207:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768085.171210:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768085.171212:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000400:00000010:1.0:1775768085.171214:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000200:1.0:1775768085.171217:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294272384, portal 4 00000100:00000001:1.0:1775768085.171218:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768085.171219:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641209088 : -112966068342528 : ffff9942099ff100) 00000100:00000040:1.0:1775768085.171222:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ff100 x1862022294272384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768110 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768085.171227:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768085.171230:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768085.171232:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768085.171234:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294272384, offset 0 00000400:00000200:1.0:1775768085.171238:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768085.171243:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768085.171247:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768085.171249:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768085.171252:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768085.171254:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768085.171257:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768085.171260:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225301:225301:256:4294967295] 192.168.202.156@tcp LPNI seq info [225297:225297:8:4294967295] 00000400:00000200:1.0:1775768085.171264:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768085.171271:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768085.171276:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768085.171279:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000800:00000200:1.0:1775768085.171284:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768085.171287:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775768085.171291:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768085.171294:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000100:1.0:1775768085.171298:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768085.171300:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768085.171304:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768085.171307:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768085.171310:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768085.171312:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768085.171314:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768085.171318:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ff100 x1862022294272384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768110 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768085.171326:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768085.171329:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ff100 x1862022294272384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768110 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768085.171336:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768085.171337:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768085.171340:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768085.171343:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:1.0:1775768085.171347:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768085.171349:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300010019 00000100:00000001:1.0:1775768085.171352:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300011325 00000100:00000001:1.0:1775768085.171354:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300010019 00000100:00000001:1.0:1775768085.171356:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300011325 00000100:00000001:1.0:1775768085.171358:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768085.171360:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768085.171362:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768085.171365:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768085/real 1775768085] req@ffff9942099ff100 x1862022294272384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768085.171372:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768085.171374:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768085.171376:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768085.171378:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768085.171380:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768085.171382:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768085.171385:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ff100 x1862022294272384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768085.171394:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ff100 x1862022294272384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768085.171402:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768085.171403:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768085.171404:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768085.171405:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768085.171408:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ff100 x1862022294272384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768085.171412:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768085.171415:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ff100 x1862022294272384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768110 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768085.171419:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768085.171420:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768085.171422:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768085.171422:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768085.171424:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768085.171426:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768085.171428:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768085.171430:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768085.171431:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768085.171433:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ff100 x1862022294272384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768110 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768085.171439:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294272384:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768085.171441:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768085.171443:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768085.171445:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ff800 x1862022294272128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768110 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768085.171449:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768085.171451:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768085.171452:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543e800. 02000000:00000001:1.0:1775768085.171454:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768085.171455:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768085.171456:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:1.0:1775768085.171457:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768085.171459:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028400. 02000000:00000001:1.0:1775768085.171461:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768085.171462:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768085.171463:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ff800. 00000100:00000001:1.0:1775768085.171465:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768085.171466:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768085.171468:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768085.171469:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fd500 x1862022294272256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768110 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768085.171474:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768085.171475:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768085.171475:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543fc00. 02000000:00000001:1.0:1775768085.171477:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768085.171477:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768085.171478:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:1.0:1775768085.171479:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768085.171480:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029800. 02000000:00000001:1.0:1775768085.171481:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768085.171482:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768085.171483:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fd500. 00000100:00000001:1.0:1775768085.171484:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768085.171485:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768085.171485:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768085.171487:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ff100 x1862022294272384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768110 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768085.171491:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768085.171492:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768085.171493:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543dc00. 02000000:00000001:1.0:1775768085.171494:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768085.171495:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768085.171496:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:1.0:1775768085.171497:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768085.171498:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002a800. 02000000:00000001:1.0:1775768085.171499:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768085.171500:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768085.171500:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ff100. 00000100:00000001:1.0:1775768085.171501:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768085.171502:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768085.171503:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768085.171506:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768085.171508:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768085.171509:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768085.171510:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768085.171511:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768085.171513:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fc700 x1862022294272512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768085.171517:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768085.171518:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768085.171519:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768085.171520:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768085.171522:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768085.171524:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294272512:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768085.171526:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768085.171527:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768085.171528:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768085.171529:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768085.171531:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543dc00. 02000000:00000001:1.0:1775768085.171533:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768085.171535:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768085.171537:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000400:00000010:1.0:1775768085.171539:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000200:1.0:1775768085.171541:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294272512, portal 4 00000100:00000001:1.0:1775768085.171543:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768085.171544:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641198336 : -112966068353280 : ffff9942099fc700) 00000100:00000040:1.0:1775768085.171548:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fc700 x1862022294272512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768110 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768085.171553:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768085.171554:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768085.171556:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768085.171557:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294272512, offset 0 00000400:00000200:1.0:1775768085.171560:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768085.171562:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768085.171564:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768085.171567:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768085.171568:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768085.171570:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768085.171571:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768085.171574:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225302:225302:256:4294967295] 192.168.202.156@tcp LPNI seq info [225298:225298:8:4294967295] 00000400:00000200:1.0:1775768085.171577:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768085.171581:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768085.171583:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768085.171586:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000800:00000200:1.0:1775768085.171588:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768085.171590:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775768085.171593:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768085.171602:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000100:1.0:1775768085.171605:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768085.171607:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768085.171609:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768085.171611:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768085.171613:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768085.171614:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768085.171616:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768085.171618:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fc700 x1862022294272512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768110 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768085.171628:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768085.171630:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fc700 x1862022294272512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768110 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768085.171634:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768085.171635:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768085.171637:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768085.171639:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:1.0:1775768085.171641:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768085.171642:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300010022 00000100:00000001:1.0:1775768085.171644:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300011325 00000100:00000001:1.0:1775768085.171645:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300010022 00000100:00000001:1.0:1775768085.171647:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300011325 00000100:00000001:1.0:1775768085.171648:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768085.171650:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768085.171651:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768085.171653:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768085/real 1775768085] req@ffff9942099fc700 x1862022294272512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768085.171657:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768085.171658:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768085.171659:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768085.171661:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768085.171662:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768085.171663:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768085.171665:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fc700 x1862022294272512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768085.171670:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fc700 x1862022294272512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768085.171674:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768085.171675:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768085.171676:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768085.171677:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768085.171679:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fc700 x1862022294272512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768085.171683:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768085.171686:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fc700 x1862022294272512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768110 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768085.171690:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768085.171691:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768085.171692:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768085.171693:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768085.171694:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768085.171696:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768085.171697:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768085.171699:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768085.171701:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768085.171703:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fc700 x1862022294272512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768110 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768085.171707:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294272512:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768085.171710:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768085.171711:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768085.171713:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fc700 x1862022294272512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768110 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768085.171717:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768085.171718:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768085.171719:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543dc00. 02000000:00000001:1.0:1775768085.171720:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768085.171721:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768085.171722:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:1.0:1775768085.171723:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768085.171724:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002bc00. 02000000:00000001:1.0:1775768085.171726:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768085.171727:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768085.171728:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fc700. 00000100:00000001:1.0:1775768085.171729:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768085.171730:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768085.171731:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768085.171733:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768085.171734:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768085.171735:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768085.171737:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768085.233255:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768085.233259:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768085.233264:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768085.233265:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768085.233268:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768085.233269:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768085.233271:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768085.233272:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768085.233274:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768085.233275:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768085.233338:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768085.233343:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768085.233346:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768085.297254:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768085.297259:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768085.297261:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768085.297265:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768085.297268:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768085.297272:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768085.297274:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768085.297276:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768085.297278:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768086.193208:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768086.193222:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768086.193224:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768086.193229:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768086.193232:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768086.193236:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768086.193238:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768086.193240:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768086.193242:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768086.193335:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768086.193339:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768086.193341:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768086.193344:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768086.193347:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768086.193349:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768086.193351:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768086.193353:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768086.193354:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768086.193356:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768086.193359:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768086.193363:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768086.193366:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775768086.193371:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768086.193373:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768086.193376:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768086.193378:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768086.193380:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768086.257237:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768086.257241:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768086.257245:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768086.257248:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768086.257250:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768086.257252:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768086.257255:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768086.257257:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768086.257259:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768086.257262:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768086.257266:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768086.257268:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768086.257269:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768086.257271:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768086.257272:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768086.257274:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768086.257275:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768086.257277:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768086.257281:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768086.257282:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768086.257284:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768086.257285:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768086.257287:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768086.257288:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768086.257290:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768086.257292:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768086.257295:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768086.257297:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768086.257298:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768086.257300:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768086.257301:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768086.257303:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768086.257305:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768086.257307:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768086.257354:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768086.257358:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768086.257361:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768086.321264:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768086.321271:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768086.321273:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768086.321278:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768086.321282:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768086.321286:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768086.321289:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768086.321291:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768086.321294:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768086.513228:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4300011325, at 4300011328 00000100:00000040:0.0:1775768086.513233:0:14953:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4300011325, at 4300011328 00000100:00000010:1.0:1775768086.513245:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cba360. 00000100:00000010:0.0:1775768086.513248:0:14953:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cbaae0. 00000100:00000001:3.0:1775768087.217206:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768087.217211:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768087.217213:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768087.217218:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768087.217221:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768087.217225:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768087.217227:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768087.217229:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768087.217232:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768087.217250:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768087.217251:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768087.217257:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768087.217257:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768087.217261:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768087.217261:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768087.217267:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:0.0:1775768087.217268:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768087.217272:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768087.217273:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768087.217276:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768087.217277:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768087.217281:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768087.217284:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768087.217287:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768087.217294:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768087.217297:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768087.217304:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768087.281254:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768087.281259:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768087.281263:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768087.281265:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768087.281267:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768087.281269:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768087.281272:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768087.281273:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768087.281276:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768087.281277:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768087.281323:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768087.281326:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768087.281329:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768087.345275:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768087.345282:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768087.345284:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768087.345299:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768087.345306:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768087.345311:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768087.345314:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768087.345317:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768087.345320:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768088.241282:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768088.241282:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768088.241283:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768088.241291:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768088.241291:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768088.241291:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768088.241296:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768088.241296:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768088.241296:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768088.241304:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:0.0:1775768088.241305:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775768088.241305:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775768088.241308:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768088.241310:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768088.241312:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768088.241313:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768088.241315:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768088.241319:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768088.241322:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768088.241323:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768088.241327:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768088.241328:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768088.241332:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768088.241336:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768088.241338:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768088.241342:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768088.241345:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768088.305254:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768088.305259:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768088.305263:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768088.305264:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768088.305266:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768088.305267:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768088.305269:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768088.305270:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768088.305272:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768088.305273:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768088.305306:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768088.305310:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768088.305312:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768088.369246:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768088.369254:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768088.369256:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768088.369263:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768088.369267:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768088.369271:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768088.369274:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768088.369277:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768088.369280:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768089.265275:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768089.265275:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768089.265282:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768089.265283:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768089.265284:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768089.265285:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768089.265287:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768089.265294:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:2.0:1775768089.265295:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775768089.265295:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:0.0:1775768089.265296:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768089.265307:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768089.265308:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:0.0:1775768089.265309:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768089.265312:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768089.265314:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768089.265316:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768089.265318:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768089.265325:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768089.265326:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768089.265340:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768089.265341:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768089.265346:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768089.265350:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768089.265351:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768089.265354:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768089.265358:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768089.329328:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768089.329334:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768089.329340:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768089.329342:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768089.329346:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768089.329347:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768089.329351:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768089.329352:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768089.329355:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768089.329357:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768089.329412:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768089.329415:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768089.329418:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768089.393272:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768089.393279:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768089.393281:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768089.393286:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768089.393290:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768089.393294:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768089.393296:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768089.393298:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768089.393300:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768090.289301:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768090.289310:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768090.289314:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768090.289323:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775768090.289324:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:0.0:1775768090.289330:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775768090.289332:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768090.289332:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768090.289337:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768090.289340:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768090.289347:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775768090.289350:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1775768090.289352:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775768090.289353:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768090.289356:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768090.289359:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1775768090.289360:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768090.289363:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768090.289366:0:36:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775768090.289366:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768090.289369:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768090.289372:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768090.289376:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768090.289378:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775768090.289380:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768090.289381:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768090.289391:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5342 00000100:00100000:1.0:1775768090.289393:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768090.289398:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768090.289403:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768090.289405:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000400:00000200:0.0:1775768090.289408:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768090.289408:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768090.289414:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:3.0:1775768090.289414:0:13471:0:(rw.c:1481:ll_writepages()) Process entered 00000400:00000200:0.0:1775768090.289422:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000080:00000001:3.0:1775768090.289424:0:13471:0:(file.c:5250:cl_sync_file_range()) Process entered 00000100:00080000:0.0:1775768090.289429:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000020:00000001:3.0:1775768090.289431:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00000040:0.0:1775768090.289438:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768090.289442:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000010:3.0:1775768090.289446:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff99420693e000. 00000100:00000040:0.0:1775768090.289448:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768090.289455:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000010:3.0:1775768090.289456:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcf800. 00000020:00000010:3.0:1775768090.289465:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210dcf200. 00000020:00000001:0.0:1775768090.289466:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768090.289472:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000010:3.0:1775768090.289474:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207981de0. 00000020:00000001:0.0:1775768090.289480:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768090.289483:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000008:00000010:3.0:1775768090.289485:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994208b50000. 00000020:00000001:0.0:1775768090.289489:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:0.0:1775768090.289495:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:0.0:1775768090.289500:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00020000:00000010:3.0:1775768090.289501:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994218000300. 00000100:00000001:0.0:1775768090.289503:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768090.289507:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00008000:00000010:3.0:1775768090.289508:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994207e08e20. 00000100:00000001:0.0:1775768090.289514:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000080:00000010:3.0:1775768090.289517:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99420541b1f8. 00000020:00000040:0.0:1775768090.289519:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000080:00000010:3.0:1775768090.289522:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077c0248. 00000020:00000010:3.0:1775768090.289526:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcf000. 00000100:00080000:0.0:1775768090.289528:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000008:00000010:3.0:1775768090.289532:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942086f8420. 00000100:00000001:0.0:1775768090.289533:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:3.0:1775768090.289537:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994207e8e400. 02000000:00000001:0.0:1775768090.289539:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00008000:00000010:3.0:1775768090.289542:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994209270a90. 02000000:00000001:0.0:1775768090.289545:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000010:3.0:1775768090.289546:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942090e33c0. 00000020:00000001:3.0:1775768090.289551:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107590086664 : -112966119464952 : ffff99420693e008) 02000000:00000001:0.0:1775768090.289553:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:3.0:1775768090.289555:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff99420693e000 00000100:00000001:0.0:1775768090.289562:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000020:00000001:3.0:1775768090.289563:0:13471:0:(cl_object.c:151:cl_object_top()) ffff9942078a76c0 -> ffff9942078a76c0 00000020:00000001:3.0:1775768090.289568:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000100:00000001:0.0:1775768090.289569:0:36:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:3.0:1775768090.289571:0:13471:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:3.0:1775768090.289576:0:13471:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:3.0:1775768090.289580:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:3.0:1775768090.289581:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:3.0:1775768090.289583:0:13471:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768090.289585:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fe300. 00020000:00000002:3.0:1775768090.289588:0:13471:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017a:0x0]io ffff9942077c02a8 type 6 ignore/verify layout 1/0 00020000:00000002:3.0:1775768090.289592:0:13471:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207c1f9b0) owner 0000000000000000/ffff994220080000 00000020:00000040:0.0:1775768090.289593:0:36:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 00020000:00000001:3.0:1775768090.289597:0:13471:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 02000000:00000001:0.0:1775768090.289600:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00020000:00000001:3.0:1775768090.289600:0:13471:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:3.0:1775768090.289603:0:13471:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 02000000:00000001:0.0:1775768090.289604:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768090.289605:0:13471:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768090.289614:0:13471:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768090.289615:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00020000:00000001:3.0:1775768090.289617:0:13471:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:3.0:1775768090.289620:0:13471:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768090.289621:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000020:00000001:3.0:1775768090.289622:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:3.0:1775768090.289623:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:3.0:1775768090.289627:0:13471:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00020000:00000002:3.0:1775768090.289628:0:13471:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207c1f9b0) owner 0000000000000000/ffff994220080000 00000020:00000001:3.0:1775768090.289630:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768090.289631:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768090.289633:0:13471:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:3.0:1775768090.289637:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00020000:00000001:3.0:1775768090.289639:0:13471:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00020000:00200000:3.0:1775768090.289642:0:13471:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 02000000:00000010:0.0:1775768090.289645:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220028000. 00020000:00000002:3.0:1775768090.289646:0:13471:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:3.0:1775768090.289653:0:13471:0:(lov_io.c:155:lov_sub_get()) Process entered 00020000:00000001:3.0:1775768090.289655:0:13471:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000020:00000001:3.0:1775768090.289656:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00000001:0.0:1775768090.289658:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1775768090.289659:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff99420693ecf0. 00000020:00000010:3.0:1775768090.289666:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dce000. 00000020:00000010:3.0:1775768090.289669:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210dcf400. 00000020:00000010:3.0:1775768090.289672:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942079817e0. 00000100:00100000:0.0:1775768090.289677:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fe300 x1862022294272640/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000008:00000010:3.0:1775768090.289678:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994208b51650. 00020000:00000010:3.0:1775768090.289683:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994218000000. 00008000:00000010:3.0:1775768090.289686:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994207e0ade8. 00000080:00000010:3.0:1775768090.289691:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99420541b0a8. 00000080:00000010:3.0:1775768090.289697:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077c1da8. 00000020:00000010:3.0:1775768090.289699:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcea00. 00000008:00000010:3.0:1775768090.289701:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942086f8160. 00020000:00000010:3.0:1775768090.289702:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994207e8ed00. 00008000:00000010:3.0:1775768090.289704:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994209270ef0. 00000080:00000010:3.0:1775768090.289711:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942090e3960. 00000020:00000001:3.0:1775768090.289718:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107590089976 : -112966119461640 : ffff99420693ecf8) 00000020:00001000:3.0:1775768090.289728:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff99420693ecf0 00000100:00000001:0.0:1775768090.289731:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000020:00000001:3.0:1775768090.289731:0:13471:0:(cl_object.c:151:cl_object_top()) ffff9942078a7788 -> ffff9942078a76c0 00000020:00000001:3.0:1775768090.289734:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:3.0:1775768090.289737:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000100:00000001:0.0:1775768090.289738:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768090.289739:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:3.0:1775768090.289742:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768090.289745:0:13471:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768090.289747:0:13471:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107612435560 : -112966097116056 : ffff994207e8e468) 00000020:00000001:3.0:1775768090.289751:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000100:00000040:0.0:1775768090.289752:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008a1e3639] to pc [ptlrpcd_rcv+-1] req@ffff9942099fe300 x1862022294272640/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775768090.289754:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768090.289760:0:13471:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768090.289766:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768090.289770:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00020000:00000001:3.0:1775768090.289774:0:13471:0:(lov_io.c:1108:lov_io_lock()) Process entered 00020000:00000001:3.0:1775768090.289779:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775768090.289781:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:3.0:1775768090.289784:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:3.0:1775768090.289788:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768090.289790:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768090.289792:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768090.289794:0:13471:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768090.289795:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:3.0:1775768090.289796:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768090.289797:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768090.289799:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00020000:00000001:3.0:1775768090.289801:0:13471:0:(lov_io.c:1114:lov_io_start()) Process entered 00020000:00000001:3.0:1775768090.289802:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775768090.289802:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00000008:00000001:3.0:1775768090.289805:0:13471:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000100:00000001:0.0:1775768090.289806:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1775768090.289808:0:13471:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000100:00080000:0.0:1775768090.289816:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000008:00000001:3.0:1775768090.289818:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:3.0:1775768090.289821:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:3.0:1775768090.289827:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1775768090.289828:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000008:00000001:3.0:1775768090.289832:0:13471:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000100:00000001:0.0:1775768090.289833:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000008:00000001:3.0:1775768090.289835:0:13471:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000008:00000001:3.0:1775768090.289839:0:13471:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107884691392 : -112965824860224 : ffff994218232fc0) 00000100:00080000:0.0:1775768090.289840:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768090.289841:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00000020:3.0:1775768090.289845:0:13471:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994218232fc0 ready 0|-|- wr 1|-|+ rd 0|- 10 in flight 00000100:00000001:1.0:1775768090.289848:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768090.289850:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775768090.289853:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000008:00000001:3.0:1775768090.289858:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:3.0:1775768090.289861:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000100:00000040:1.0:1775768090.289863:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fe300 x1862022294272640/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1775768090.289866:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1775768090.289874:0:13471:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000008:00000020:3.0:1775768090.289877:0:13471:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994210675880 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000100:00080000:0.0:1775768090.289882:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5342 00000008:00000001:3.0:1775768090.289899:0:13471:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1775768090.289900:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00000001:3.0:1775768090.289911:0:13471:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768090.289912:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000001:3.0:1775768090.289915:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768090.289917:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:1.0:1775768090.289917:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768090.289923:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00020000:00000001:3.0:1775768090.289923:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768090.289925:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 02000000:00000001:1.0:1775768090.289927:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00020000:00000001:3.0:1775768090.289933:0:13471:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768090.289934:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 02000000:00000001:1.0:1775768090.289934:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768090.289946:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:3.0:1775768090.289946:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768090.289948:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294272640:192.168.202.156@tcp:250:kworker.0 00000100:00000040:0.0:1775768090.289956:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000001:3.0:1775768090.289956:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00000100:00000001:1.0:1775768090.289957:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1775768090.289964:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00020000:00000001:3.0:1775768090.289964:0:13471:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 02000000:00000001:1.0:1775768090.289965:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00020000:00000001:3.0:1775768090.289971:0:13471:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000020:00000001:0.0:1775768090.289972:0:36:0:(genops.c:811:class_conn2export()) Process entered 02000000:00000001:1.0:1775768090.289972:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768090.289976:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000040:0.0:1775768090.289978:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 02000000:00000001:1.0:1775768090.289978:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:3.0:1775768090.289981:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00000020:00000001:0.0:1775768090.289983:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000010:1.0:1775768090.289983:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543dc00. 00020000:00000001:3.0:1775768090.289984:0:13471:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1775768090.289988:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 02000000:00000001:1.0:1775768090.289990:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768090.289993:0:13471:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000020:00000001:0.0:1775768090.289997:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:3.0:1775768090.289998:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00000400:00000200:1.0:1775768090.290005:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:3.0:1775768090.290005:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000020:00000001:0.0:1775768090.290006:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000400:00000010:1.0:1775768090.290011:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00020000:00000001:3.0:1775768090.290012:0:13471:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00000100:00000040:0.0:1775768090.290016:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00020000:00000001:3.0:1775768090.290018:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000001:0.0:1775768090.290033:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:1.0:1775768090.290034:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000020:00000001:3.0:1775768090.290037:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000100:00000040:0.0:1775768090.290042:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000001:3.0:1775768090.290042:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000100:00000200:1.0:1775768090.290043:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294272640, portal 25 00020000:00000001:3.0:1775768090.290047:0:13471:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768090.290052:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768090.290055:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00020000:00000001:3.0:1775768090.290055:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768090.290056:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641205504 : -112966068346112 : ffff9942099fe300) 00020000:00000001:3.0:1775768090.290062:0:13471:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000020:00000040:0.0:1775768090.290066:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000020:00000001:3.0:1775768090.290067:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000100:00000040:1.0:1775768090.290071:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fe300 x1862022294272640/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768115 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768090.290076:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000020:00000001:3.0:1775768090.290079:0:13471:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000020:00000001:3.0:1775768090.290085:0:13471:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000100:00000001:0.0:1775768090.290087:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768090.290093:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 02000000:00000001:0.0:1775768090.290094:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1775768090.290094:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00020000:00000001:3.0:1775768090.290100:0:13471:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00000100:00000040:1.0:1775768090.290102:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00020000:00000001:3.0:1775768090.290104:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 02000000:00000001:0.0:1775768090.290105:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:1.0:1775768090.290106:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000020:00000001:3.0:1775768090.290110:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000100:00000200:1.0:1775768090.290111:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294272640, offset 0 02000000:00000001:0.0:1775768090.290112:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768090.290115:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000100:00000001:0.0:1775768090.290119:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00020000:00000001:3.0:1775768090.290120:0:13471:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768090.290123:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000008:00100000:0.0:1775768090.290125:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00020000:00000001:3.0:1775768090.290130:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1775768090.290131:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768090.290139:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:3.0:1775768090.290143:0:13471:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000100:00000001:0.0:1775768090.290144:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768090.290149:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000400:00000200:1.0:1775768090.290150:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00000010:0.0:1775768090.290152:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fd880. 00000400:00000200:1.0:1775768090.290179:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768090.290182:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768090.290185:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768090.290188:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000020:00000040:0.0:1775768090.290193:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000400:00000200:1.0:1775768090.290196:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225303:225303:256:4294967295] 192.168.202.156@tcp LPNI seq info [225299:225299:8:4294967295] 02000000:00000001:0.0:1775768090.290198:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768090.290201:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768090.290203:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:3.0:1775768090.290204:0:13471:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768090.290205:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768090.290208:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000020:00000001:3.0:1775768090.290213:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 02000000:00000001:0.0:1775768090.290214:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768090.290218:0:13471:0:(lov_io.c:675:lov_io_fini()) Process entered 02000000:00000010:0.0:1775768090.290221:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220029400. 00000400:00000200:1.0:1775768090.290223:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00020000:00000001:3.0:1775768090.290225:0:13471:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000100:00000001:0.0:1775768090.290228:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768090.290229:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:3.0:1775768090.290236:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000100:00100000:0.0:1775768090.290240:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fd880 x1862022294272768/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.0:1775768090.290240:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000020:00001000:3.0:1775768090.290242:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff99420693ecf0 00000020:00000010:3.0:1775768090.290254:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210dcf400. 00000800:00000010:1.0:1775768090.290255:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000100:00000001:0.0:1775768090.290259:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768090.290264:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1775768090.290266:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942079817e0. 00000100:00000040:0.0:1775768090.290273:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007f8a5f3b] to pc [ptlrpcd_rcv+-1] req@ffff9942099fd880 x1862022294272768/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1775768090.290274:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000008:00000010:3.0:1775768090.290274:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994208b51650. 00000800:00000200:1.0:1775768090.290286:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00020000:00000010:3.0:1775768090.290287:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994218000000. 00000100:00000001:0.0:1775768090.290290:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:3.0:1775768090.290295:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994207e0ade8. 00000100:00080000:0.0:1775768090.290297:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000080:00000010:3.0:1775768090.290300:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99420541b0a8. 00000100:00080000:0.0:1775768090.290301:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000200:1.1:1775768090.290301:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:0.0:1775768090.290310:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000800:00000200:1.1:1775768090.290312:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000100:00080000:0.0:1775768090.290313:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000080:00000010:3.0:1775768090.290313:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077c1da8. 00000800:00000100:1.0:1775768090.290325:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000020:00000010:3.0:1775768090.290326:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dce000. 00000100:00000001:0.0:1775768090.290327:0:36:0:(import.c:484:import_select_connection()) Process entered 00000800:00000010:1.0:1775768090.290331:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000100:00080000:0.0:1775768090.290337:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5342 00000008:00000010:3.0:1775768090.290337:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942086f8160. 00000400:00000200:1.0:1775768090.290341:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00020000:00000010:3.0:1775768090.290350:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994207e8ed00. 00000400:00000200:0.0:1775768090.290351:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00008000:00000010:3.0:1775768090.290358:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994209270ef0. 00000400:00000200:1.0:1775768090.290359:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768090.290365:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000080:00000010:3.0:1775768090.290367:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942090e3960. 00000400:00000200:1.0:1775768090.290369:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000100:00080000:0.0:1775768090.290370:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000020:00000010:3.0:1775768090.290375:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcea00. 00000400:00000010:1.0:1775768090.290376:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000040:0.0:1775768090.290380:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000010:3.0:1775768090.290381:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff99420693ecf0. 00000100:00000001:1.0:1775768090.290386:0:2399:0:(events.c:34:request_out_callback()) Process entered 00020000:00000001:3.0:1775768090.290389:0:13471:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00000100:00000001:0.0:1775768090.290390:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:1.0:1775768090.290396:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fe300 x1862022294272640/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768115 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:3.0:1775768090.290397:0:13471:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000100:00000040:0.0:1775768090.290398:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000080:00000001:3.0:1775768090.290403:0:13471:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000100:00000001:0.0:1775768090.290404:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000080:00200000:3.0:1775768090.290406:0:13471:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000020:00000001:0.0:1775768090.290411:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768090.290415:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:0.0:1775768090.290423:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000080:00000001:3.0:1775768090.290423:0:13471:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:3.0:1775768090.290429:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00000040:0.0:1775768090.290431:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:0.0:1775768090.290434:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00001000:3.0:1775768090.290436:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff99420693e000 00000020:00000001:0.0:1775768090.290437:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000001:1.0:1775768090.290437:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000020:00000010:3.0:1775768090.290442:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210dcf200. 00000100:00000040:0.0:1775768090.290449:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000040:1.0:1775768090.290450:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fe300 x1862022294272640/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768115 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1775768090.290450:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207981de0. 00000100:00000001:0.0:1775768090.290459:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000008:00000010:3.0:1775768090.290459:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994208b50000. 00000100:00000040:0.0:1775768090.290470:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00020000:00000010:3.0:1775768090.290470:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994218000300. 00000100:00000001:1.0:1775768090.290474:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:3.0:1775768090.290478:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994207e08e20. 00000100:00000001:0.0:1775768090.290479:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:1.0:1775768090.290479:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000080:00000010:3.0:1775768090.290484:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99420541b1f8. 00000020:00000040:0.0:1775768090.290485:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00000001:1.0:1775768090.290490:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:3.0:1775768090.290492:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077c0248. 00000100:00080000:0.0:1775768090.290495:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000020:00000010:3.0:1775768090.290502:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcf800. 00000100:00000001:0.0:1775768090.290503:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768090.290503:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00000010:3.0:1775768090.290513:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942086f8420. 02000000:00000001:0.0:1775768090.290514:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1775768090.290515:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4300016446 02000000:00000001:0.0:1775768090.290522:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000010:3.0:1775768090.290522:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994207e8e400. 00000100:00000010:1.0:1775768090.290527:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba360. 02000000:00000001:0.0:1775768090.290530:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:3.0:1775768090.290530:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994209270a90. 00000100:00000001:0.0:1775768090.290536:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000080:00000010:3.0:1775768090.290536:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942090e33c0. 00000008:00100000:0.0:1775768090.290543:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000020:00000010:3.0:1775768090.290543:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcf000. 00000100:00000001:1.0:1775768090.290544:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000020:00000010:3.0:1775768090.290549:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff99420693e000. 00000008:00000001:0.0:1775768090.290552:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:3.0:1775768090.290554:0:13471:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768090.290555:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768090.290556:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4300016446 00000080:00000001:3.0:1775768090.290559:0:13471:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768090.290562:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ff480. 00000100:00000010:1.0:1775768090.290564:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba000. 00000020:00000040:0.0:1775768090.290566:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000100:00000001:1.0:1775768090.290570:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 02000000:00000001:0.0:1775768090.290571:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000080:00000001:3.0:1775768090.290572:0:13471:0:(rw.c:1481:ll_writepages()) Process entered 00000100:00000001:1.0:1775768090.290575:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768090.290577:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:3.0:1775768090.290579:0:13471:0:(file.c:5250:cl_sync_file_range()) Process entered 00000020:00000001:3.0:1775768090.290584:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00000001:0.0:1775768090.290585:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1775768090.290585:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768090.290590:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1775768090.290591:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000020:00000010:3.0:1775768090.290592:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff99420693e000. 02000000:00000001:0.0:1775768090.290593:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1775768090.290600:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcf000. 00000100:00100000:1.0:1775768090.290601:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768090/real 1775768090] req@ffff9942099fe300 x1862022294272640/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1775768090.290602:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002bc00. 00000020:00000010:3.0:1775768090.290610:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210dcf800. 00000100:00000001:0.0:1775768090.290611:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1775768090.290621:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207981de0. 00000400:00000001:1.0:1775768090.290624:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000008:00000010:3.0:1775768090.290625:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994208b50000. 00000100:00100000:0.0:1775768090.290629:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ff480 x1862022294272896/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:1.0:1775768090.290629:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00020000:00000010:3.0:1775768090.290631:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994218000300. 00000400:00000010:1.0:1775768090.290634:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00008000:00000010:3.0:1775768090.290640:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994207e08e20. 00000400:00000200:1.0:1775768090.290642:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768090.290649:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000080:00000010:3.0:1775768090.290650:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99420541b1f8. 00000080:00000010:3.0:1775768090.290659:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077c0248. 00000100:00000001:0.0:1775768090.290660:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775768090.290661:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1775768090.290667:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1775768090.290667:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcf200. 00000100:00000200:1.0:1775768090.290671:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fe300 x1862022294272640/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000010:3.0:1775768090.290678:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942086f8420. 00000100:00000040:0.0:1775768090.290686:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e73dcd12] to pc [ptlrpcd_rcv+-1] req@ffff9942099ff480 x1862022294272896/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000010:3.0:1775768090.290687:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994207e8e400. 00008000:00000010:3.0:1775768090.290696:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994209270a90. 00000080:00000010:3.0:1775768090.290705:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942090e33c0. 00000020:00000001:3.0:1775768090.290714:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107590086664 : -112966119464952 : ffff99420693e008) 00000020:00001000:3.0:1775768090.290725:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff99420693e000 00000100:00000001:0.0:1775768090.290727:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775768090.290727:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fe300 x1862022294272640/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775768090.290734:0:13471:0:(cl_object.c:151:cl_object_top()) ffff9942078a7f58 -> ffff9942078a7f58 00000100:00080000:0.0:1775768090.290736:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000001:3.0:1775768090.290741:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:3.0:1775768090.290746:0:13471:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:3.0:1775768090.290752:0:13471:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017b:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000100:00080000:0.0:1775768090.290753:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775768090.290760:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768090.290765:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000020:00000001:3.0:1775768090.290767:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:3.0:1775768090.290772:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00000001:0.0:1775768090.290774:0:36:0:(import.c:484:import_select_connection()) Process entered 00000080:00000001:3.0:1775768090.290778:0:13471:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768090.290781:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00080000:0.0:1775768090.290783:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5342 00020000:00000002:3.0:1775768090.290783:0:13471:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017b:0x0]io ffff9942077c02a8 type 6 ignore/verify layout 1/0 00000100:00000001:1.0:1775768090.290785:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768090.290793:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000400:00000200:0.0:1775768090.290795:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768090.290797:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00020000:00000002:3.0:1775768090.290802:0:13471:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207c1f000) owner 0000000000000000/ffff994220080000 00000400:00000200:0.0:1775768090.290805:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00100000:1.0:1775768090.290810:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fe300 x1862022294272640/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768090.290814:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00020000:00000001:3.0:1775768090.290814:0:13471:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:3.0:1775768090.290821:0:13471:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00000100:00000040:0.0:1775768090.290824:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00020000:00000001:3.0:1775768090.290826:0:13471:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00000100:00000001:0.0:1775768090.290833:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00020000:00000001:3.0:1775768090.290833:0:13471:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768090.290840:0:13471:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1775768090.290847:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00020000:00000001:3.0:1775768090.290850:0:13471:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00000100:00000001:1.0:1775768090.290852:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00020000:00000001:3.0:1775768090.290854:0:13471:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768090.290856:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775768090.290861:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:3.0:1775768090.290866:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:0.0:1775768090.290867:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000040:1.0:1775768090.290867:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fe300 x1862022294272640/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768115 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:3.0:1775768090.290869:0:13471:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00000020:00000040:0.0:1775768090.290871:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00020000:00000002:3.0:1775768090.290874:0:13471:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207c1f000) owner 0000000000000000/ffff994220080000 00000020:00000001:0.0:1775768090.290879:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768090.290887:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:3.0:1775768090.290887:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768090.290899:0:13471:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:0.0:1775768090.290900:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:1.0:1775768090.290900:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:3.0:1775768090.290904:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00020000:00000001:3.0:1775768090.290908:0:13471:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00000020:00000001:0.0:1775768090.290910:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:1.0:1775768090.290910:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00020000:00200000:3.0:1775768090.290911:0:13471:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00000001:1.0:1775768090.290921:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00020000:00000002:3.0:1775768090.290923:0:13471:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00000100:00000001:1.0:1775768090.290924:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768090.290928:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768090.290933:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768090.290939:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775768090.290939:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00020000:00000001:3.0:1775768090.290940:0:13471:0:(lov_io.c:155:lov_sub_get()) Process entered 00020000:00000001:3.0:1775768090.290946:0:13471:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000100:00000040:0.0:1775768090.290948:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00080000:1.0:1775768090.290948:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000020:00000001:3.0:1775768090.290950:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00000001:0.0:1775768090.290958:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000010:3.0:1775768090.290959:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff99420693ecf0. 00000100:00080000:1.0:1775768090.290962:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000020:00000040:0.0:1775768090.290970:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000020:00000010:3.0:1775768090.290970:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcea00. 00000100:00000001:1.0:1775768090.290972:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000010:3.0:1775768090.290979:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210dce000. 00000100:00080000:0.0:1775768090.290983:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000040:1.0:1775768090.290984:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fe300 x1862022294272640/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768115 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1775768090.290987:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942079817e0. 00000100:00000001:0.0:1775768090.290990:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:3.0:1775768090.290996:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994208b51650. 02000000:00000001:0.0:1775768090.290997:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00020000:00000010:3.0:1775768090.291005:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994218000000. 02000000:00000001:0.0:1775768090.291006:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768090.291015:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:3.0:1775768090.291015:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994207e0ade8. 00000100:00100000:1.0:1775768090.291016:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294272640:192.168.202.156@tcp:250:kworker.0 00000100:00000001:0.0:1775768090.291034:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775768090.291038:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000080:00000010:3.0:1775768090.291041:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99420541b0a8. 00000008:00000001:0.0:1775768090.291047:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768090.291047:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000080:00000010:3.0:1775768090.291048:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077c1da8. 00000100:00000001:1.0:1775768090.291061:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000020:00000010:3.0:1775768090.291061:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcf400. 00000100:00000001:0.0:1775768090.291062:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768090.291070:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fe300 x1862022294272640/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768115 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1775768090.291073:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fd180. 00000008:00000010:3.0:1775768090.291074:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942086f8160. 00000020:00000040:0.0:1775768090.291084:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00020000:00000010:3.0:1775768090.291084:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994207e8ed00. 00000100:00000001:1.0:1775768090.291085:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768090.291089:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768090.291094:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543dc00. 00008000:00000010:3.0:1775768090.291094:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994209270ef0. 02000000:00000001:0.0:1775768090.291095:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768090.291100:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768090.291104:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000080:00000010:3.0:1775768090.291105:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942090e3960. 00000100:00000001:0.0:1775768090.291106:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000020:00000001:1.0:1775768090.291107:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768090.291112:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 02000000:00000001:0.0:1775768090.291113:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000020:00000001:3.0:1775768090.291114:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107590089976 : -112966119461640 : ffff99420693ecf8) 00000020:00000001:1.0:1775768090.291116:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000001:0.0:1775768090.291117:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775768090.291120:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028000. 00000020:00001000:3.0:1775768090.291130:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff99420693ecf0 02000000:00000001:1.0:1775768090.291132:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000010:0.0:1775768090.291133:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002a800. 02000000:00000001:1.0:1775768090.291139:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000020:00000001:3.0:1775768090.291139:0:13471:0:(cl_object.c:151:cl_object_top()) ffff9942078a75f8 -> ffff9942078a7f58 00000100:00000001:0.0:1775768090.291141:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775768090.291142:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fe300. 00000020:00000001:3.0:1775768090.291150:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000100:00000001:1.0:1775768090.291151:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768090.291153:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768090.291191:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fd180 x1862022294273024/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000001:3.0:1775768090.291191:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:3.0:1775768090.291195:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00000001:1.0:1775768090.291201:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1775768090.291201:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768090.291213:0:13471:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768090.291216:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768090.291220:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768090.291221:0:13471:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107612435560 : -112966097116056 : ffff994207e8e468) 00000100:00000001:1.0:1775768090.291232:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1775768090.291233:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000100:00000001:1.0:1775768090.291237:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1775768090.291238:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768090.291239:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775768090.291240:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00020000:00000001:3.0:1775768090.291245:0:13471:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768090.291246:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768090.291251:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fd880 x1862022294272768/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775768090.291255:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768090.291262:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004ec16d17] to pc [ptlrpcd_rcv+-1] req@ffff9942099fd180 x1862022294273024/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775768090.291265:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00020000:00000001:3.0:1775768090.291270:0:13471:0:(lov_io.c:1108:lov_io_lock()) Process entered 00020000:00000001:3.0:1775768090.291275:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775768090.291281:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:3.0:1775768090.291287:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:3.0:1775768090.291293:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768090.291295:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768090.291302:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768090.291302:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768090.291314:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768090.291315:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1775768090.291318:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768090.291319:0:13471:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768090.291320:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1775768090.291325:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000100:00080000:0.0:1775768090.291328:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000001:3.0:1775768090.291329:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768090.291330:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768090.291343:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768090.291345:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1775768090.291348:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294272768:192.168.202.156@tcp:38:kworker.0 00000020:00000001:3.0:1775768090.291350:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00000100:00000001:0.0:1775768090.291355:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00020000:00000001:3.0:1775768090.291355:0:13471:0:(lov_io.c:1114:lov_io_start()) Process entered 00000100:00000001:1.0:1775768090.291360:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00020000:00000001:3.0:1775768090.291362:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00080000:0.0:1775768090.291363:0:36:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:1.0:1775768090.291364:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:3.0:1775768090.291364:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 02000000:00000001:1.0:1775768090.291371:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1775768090.291371:0:13471:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000100:00000001:0.0:1775768090.291376:0:36:0:(import.c:484:import_select_connection()) Process entered 02000000:00000001:1.0:1775768090.291376:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:3.0:1775768090.291376:0:13471:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 02000000:00000010:1.0:1775768090.291381:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543dc00. 00000008:00000001:3.0:1775768090.291381:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000100:00080000:0.0:1775768090.291386:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5342 02000000:00000001:1.0:1775768090.291386:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000020:3.0:1775768090.291386:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:3.0:1775768090.291391:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1775768090.291398:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00000001:3.0:1775768090.291398:0:13471:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000400:00000200:0.0:1775768090.291402:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00000001:3.0:1775768090.291405:0:13471:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000400:00000010:1.0:1775768090.291407:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000008:00000001:3.0:1775768090.291408:0:13471:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107884688256 : -112965824863360 : ffff994218232380) 00000400:00000200:0.0:1775768090.291413:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000008:00000020:3.0:1775768090.291416:0:13471:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994218232380 ready 0|-|- wr 1|-|+ rd 0|- 9 in flight 00000400:00000010:1.0:1775768090.291417:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00080000:0.0:1775768090.291418:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000008:00000001:3.0:1775768090.291429:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000100:00000040:0.0:1775768090.291430:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000200:1.0:1775768090.291430:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294272768, portal 10 00000008:00000020:3.0:1775768090.291433:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000100:00000001:1.0:1775768090.291437:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000008:00000001:3.0:1775768090.291437:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768090.291438:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775768090.291443:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641202816 : -112966068348800 : ffff9942099fd880) 00000008:00000001:3.0:1775768090.291446:0:13471:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000100:00000040:0.0:1775768090.291450:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000008:00000020:3.0:1775768090.291450:0:13471:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994210674fc0 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000100:00000040:1.0:1775768090.291455:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fd880 x1862022294272768/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768115 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768090.291461:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768090.291475:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000008:00000001:3.0:1775768090.291475:0:13471:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:0.0:1775768090.291481:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000100:00000001:1.0:1775768090.291486:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00000001:3.0:1775768090.291486:0:13471:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768090.291487:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768090.291493:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:3.0:1775768090.291494:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768090.291500:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000020:00000001:0.0:1775768090.291505:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00020000:00000001:3.0:1775768090.291507:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775768090.291508:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000020:00000001:0.0:1775768090.291514:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00020000:00000001:3.0:1775768090.291517:0:13471:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775768090.291519:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294272768, offset 0 00000020:00000001:3.0:1775768090.291522:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768090.291524:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000001:3.0:1775768090.291532:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00000100:00000001:0.0:1775768090.291534:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00020000:00000001:3.0:1775768090.291538:0:13471:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00000400:00000200:1.0:1775768090.291539:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00020000:00000001:3.0:1775768090.291542:0:13471:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000100:00000040:0.0:1775768090.291543:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000001:3.0:1775768090.291546:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00000100:00000001:0.0:1775768090.291548:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775768090.291550:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00000400:00000200:1.0:1775768090.291554:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:3.0:1775768090.291556:0:13471:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1775768090.291558:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00020000:00000001:3.0:1775768090.291560:0:13471:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000020:00000001:3.0:1775768090.291566:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00000100:00080000:0.0:1775768090.291567:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000400:00000200:1.0:1775768090.291567:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000020:00000001:3.0:1775768090.291572:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000100:00000001:0.0:1775768090.291577:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768090.291578:0:13471:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00000400:00000200:1.0:1775768090.291582:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00020000:00000001:3.0:1775768090.291584:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 02000000:00000001:0.0:1775768090.291586:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000020:00000001:3.0:1775768090.291587:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000020:00000001:3.0:1775768090.291591:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 02000000:00000001:0.0:1775768090.291592:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1775768090.291592:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00020000:00000001:3.0:1775768090.291594:0:13471:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768090.291602:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00020000:00000001:3.0:1775768090.291603:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768090.291604:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768090.291613:0:13471:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000100:00000001:0.0:1775768090.291614:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:1.0:1775768090.291614:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000020:00000001:3.0:1775768090.291620:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:3.0:1775768090.291622:0:13471:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000008:00100000:0.0:1775768090.291624:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000020:00000001:3.0:1775768090.291626:0:13471:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000008:00000001:0.0:1775768090.291630:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768090.291631:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225304:225304:256:4294967295] 192.168.202.156@tcp LPNI seq info [225300:225300:8:4294967295] 00000020:00000001:3.0:1775768090.291631:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:3.0:1775768090.291637:0:13471:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00020000:00000001:3.0:1775768090.291641:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000001:0.0:1775768090.291642:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768090.291644:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000400:00000200:1.0:1775768090.291651:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:3.0:1775768090.291651:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000100:00000010:0.0:1775768090.291653:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fed80. 00020000:00000001:3.0:1775768090.291658:0:13471:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1775768090.291659:0:36:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00020000:00000001:3.0:1775768090.291662:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768090.291667:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00020000:00000001:3.0:1775768090.291669:0:13471:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000020:00000001:3.0:1775768090.291672:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 02000000:00000001:0.0:1775768090.291673:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768090.291674:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000020:00000001:3.0:1775768090.291678:0:13471:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768090.291680:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000020:00000001:3.0:1775768090.291684:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 02000000:00000001:0.0:1775768090.291686:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00020000:00000001:3.0:1775768090.291690:0:13471:0:(lov_io.c:675:lov_io_fini()) Process entered 02000000:00000001:0.0:1775768090.291692:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768090.291693:0:13471:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:3.0:1775768090.291699:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00000800:00000200:1.0:1775768090.291701:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 02000000:00000010:0.0:1775768090.291702:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220029800. 00000020:00000001:3.0:1775768090.291702:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:3.0:1775768090.291704:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff99420693ecf0 00000020:00000010:3.0:1775768090.291712:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210dce000. 00000100:00000001:0.0:1775768090.291714:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1775768090.291715:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000020:00000010:3.0:1775768090.291721:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942079817e0. 00000100:00100000:0.0:1775768090.291728:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fed80 x1862022294273152/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000008:00000010:3.0:1775768090.291728:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994208b51650. 00000800:00000200:1.0:1775768090.291729:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00020000:00000010:3.0:1775768090.291738:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994218000000. 00000800:00000200:1.0:1775768090.291742:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00008000:00000010:3.0:1775768090.291747:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994207e0ade8. 00000080:00000010:3.0:1775768090.291755:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99420541b0a8. 00000800:00000200:1.1:1775768090.291762:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000080:00000010:3.0:1775768090.291765:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077c1da8. 00000800:00000200:1.1:1775768090.291766:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000020:00000010:3.0:1775768090.291772:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcea00. 00000100:00000001:0.0:1775768090.291774:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000008:00000010:3.0:1775768090.291778:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942086f8160. 00000800:00000100:1.0:1775768090.291781:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000100:00000001:0.0:1775768090.291783:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:3.0:1775768090.291788:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994207e8ed00. 00000800:00000010:1.0:1775768090.291789:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000100:00000040:0.0:1775768090.291794:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000c09b4d8] to pc [ptlrpcd_rcv+-1] req@ffff9942099fed80 x1862022294273152/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00008000:00000010:3.0:1775768090.291799:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994209270ef0. 00000400:00000200:1.0:1775768090.291801:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000080:00000010:3.0:1775768090.291807:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942090e3960. 00000400:00000200:1.0:1775768090.291818:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000020:00000010:3.0:1775768090.291818:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcf400. 00000100:00000001:0.0:1775768090.291821:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1775768090.291830:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff99420693ecf0. 00000400:00000200:1.0:1775768090.291831:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000100:00000040:0.0:1775768090.291832:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5352) 00000400:00000010:1.0:1775768090.291840:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00020000:00000001:3.0:1775768090.291841:0:13471:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:3.0:1775768090.291845:0:13471:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000100:00000001:1.0:1775768090.291848:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000080:00000001:3.0:1775768090.291849:0:13471:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000080:00200000:3.0:1775768090.291852:0:13471:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00000200:1.0:1775768090.291857:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fd880 x1862022294272768/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768115 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000001:3.0:1775768090.291858:0:13471:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:3.0:1775768090.291863:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:3.0:1775768090.291868:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff99420693e000 00000020:00000010:3.0:1775768090.291876:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210dcf800. 00000100:00000001:1.0:1775768090.291878:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000020:00000010:3.0:1775768090.291879:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207981de0. 00000008:00000010:3.0:1775768090.291884:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994208b50000. 00000100:00000040:1.0:1775768090.291885:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fd880 x1862022294272768/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768115 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000010:3.0:1775768090.291891:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994218000300. 00008000:00000010:3.0:1775768090.291900:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994207e08e20. 00000100:00000001:1.0:1775768090.291909:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:3.0:1775768090.291911:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99420541b1f8. 00000100:00000001:1.0:1775768090.291912:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000080:00000010:3.0:1775768090.291918:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077c0248. 00000100:00000001:1.0:1775768090.291921:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768090.291929:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000020:00000010:3.0:1775768090.291929:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcf000. 00000400:00000200:1.0:1775768090.291937:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00000010:3.0:1775768090.291938:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942086f8420. 00000100:00000001:1.0:1775768090.291939:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300015124 00000100:00000001:1.0:1775768090.291946:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300016446 00020000:00000010:3.0:1775768090.291946:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994207e8e400. 00000100:00000001:1.0:1775768090.291954:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300015124 00008000:00000010:3.0:1775768090.291954:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994209270a90. 00000100:00000001:1.0:1775768090.291961:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300016446 00000080:00000010:3.0:1775768090.291962:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942090e33c0. 00000100:00000001:1.0:1775768090.291968:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1775768090.291968:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcf200. 00000020:00000010:3.0:1775768090.291975:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff99420693e000. 00000100:00000001:1.0:1775768090.291976:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:3.0:1775768090.291979:0:13471:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768090.291981:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000080:00000001:3.0:1775768090.291983:0:13471:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768090.291989:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768090/real 1775768090] req@ffff9942099fd880 x1862022294272768/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000001:3.0:1775768090.291990:0:13471:0:(rw.c:1481:ll_writepages()) Process entered 00000080:00000001:3.0:1775768090.291993:0:13471:0:(file.c:5250:cl_sync_file_range()) Process entered 00000020:00000001:3.0:1775768090.291996:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:3.0:1775768090.292003:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff99420693e000. 00000020:00000010:3.0:1775768090.292010:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcf200. 00000020:00000010:3.0:1775768090.292018:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210dcf000. 00000020:00000010:3.0:1775768090.292039:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207981de0. 00000400:00000001:1.0:1775768090.292045:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000008:00000010:3.0:1775768090.292046:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994208b50000. 00000400:00000001:1.0:1775768090.292053:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00020000:00000010:3.0:1775768090.292053:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994218000300. 00008000:00000010:3.0:1775768090.292059:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994207e08e20. 00000400:00000010:1.0:1775768090.292061:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000080:00000010:3.0:1775768090.292063:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99420541b1f8. 00000080:00000010:3.0:1775768090.292066:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077c0248. 00000400:00000200:1.0:1775768090.292068:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000020:00000010:3.0:1775768090.292069:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcf800. 00000400:00000010:1.0:1775768090.292073:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000008:00000010:3.0:1775768090.292074:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942086f8420. 00020000:00000010:3.0:1775768090.292082:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994207e8e400. 00000100:00000001:1.0:1775768090.292084:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00008000:00000010:3.0:1775768090.292086:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994209270a90. 00000080:00000010:3.0:1775768090.292088:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942090e33c0. 00000020:00000001:3.0:1775768090.292091:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107590086664 : -112966119464952 : ffff99420693e008) 00000100:00000200:1.0:1775768090.292093:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fd880 x1862022294272768/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00001000:3.0:1775768090.292095:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff99420693e000 00000020:00000001:3.0:1775768090.292101:0:13471:0:(cl_object.c:151:cl_object_top()) ffff99420791a850 -> ffff99420791a850 00000020:00000001:3.0:1775768090.292104:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:3.0:1775768090.292106:0:13471:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:3.0:1775768090.292110:0:13471:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017f:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:3.0:1775768090.292116:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:3.0:1775768090.292117:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:3.0:1775768090.292119:0:13471:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:3.0:1775768090.292125:0:13471:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017f:0x0]io ffff9942077c02a8 type 6 ignore/verify layout 1/0 00020000:00000002:3.0:1775768090.292130:0:13471:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207dfbd90) owner 0000000000000000/ffff994220080000 00000100:00000200:1.0:1775768090.292136:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fd880 x1862022294272768/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:3.0:1775768090.292136:0:13471:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:3.0:1775768090.292139:0:13471:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:3.0:1775768090.292142:0:13471:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00020000:00000001:3.0:1775768090.292144:0:13471:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768090.292149:0:13471:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:3.0:1775768090.292179:0:13471:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:3.0:1775768090.292180:0:13471:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768090.292183:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:3.0:1775768090.292185:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:3.0:1775768090.292188:0:13471:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00020000:00000002:3.0:1775768090.292192:0:13471:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207dfbd90) owner 0000000000000000/ffff994220080000 00000020:00000001:3.0:1775768090.292197:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768090.292198:0:13471:0:(cl_io.c:823:cl_io_loop()) Process entered 00000100:00000001:1.0:1775768090.292200:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:3.0:1775768090.292200:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00020000:00000001:3.0:1775768090.292202:0:13471:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00000100:00000001:1.0:1775768090.292205:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00020000:00200000:3.0:1775768090.292205:0:13471:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00000002:3.0:1775768090.292208:0:13471:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00000100:00000001:1.0:1775768090.292211:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00020000:00000001:3.0:1775768090.292212:0:13471:0:(lov_io.c:155:lov_sub_get()) Process entered 00020000:00000001:3.0:1775768090.292214:0:13471:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000100:00000001:1.0:1775768090.292216:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1775768090.292216:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:3.0:1775768090.292220:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff99420693ecf0. 00000020:00000010:3.0:1775768090.292224:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcf400. 00000100:00100000:1.0:1775768090.292226:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fd880 x1862022294272768/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1775768090.292229:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210dcea00. 00000020:00000010:3.0:1775768090.292236:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942079817e0. 00000008:00000010:3.0:1775768090.292241:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994208b51650. 00020000:00000010:3.0:1775768090.292245:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994218000000. 00008000:00000010:3.0:1775768090.292251:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994207e0ade8. 00000080:00000010:3.0:1775768090.292259:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99420541b0a8. 00000100:00000001:1.0:1775768090.292261:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000080:00000010:3.0:1775768090.292264:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077c1da8. 00000020:00000010:3.0:1775768090.292270:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dce000. 00000100:00000040:1.0:1775768090.292271:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fd880 x1862022294272768/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768115 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000010:3.0:1775768090.292276:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942086f8160. 00020000:00000010:3.0:1775768090.292285:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994207e8ed00. 00000100:00000001:1.0:1775768090.292295:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00008000:00000010:3.0:1775768090.292295:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994209270ef0. 00000080:00000010:3.0:1775768090.292305:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942090e3960. 00000100:00000001:1.0:1775768090.292306:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1775768090.292315:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107590089976 : -112966119461640 : ffff99420693ecf8) 00000100:00000001:1.0:1775768090.292316:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768090.292321:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00001000:3.0:1775768090.292323:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff99420693ecf0 00000100:00000001:1.0:1775768090.292324:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000020:00000001:3.0:1775768090.292329:0:13471:0:(cl_object.c:151:cl_object_top()) ffff99420791a210 -> ffff99420791a850 00000100:00000001:1.0:1775768090.292331:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:3.0:1775768090.292332:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:3.0:1775768090.292335:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:3.0:1775768090.292337:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00080000:1.0:1775768090.292338:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000020:00000001:3.0:1775768090.292341:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768090.292345:0:13471:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768090.292346:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00020000:00000001:3.0:1775768090.292353:0:13471:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107612435560 : -112966097116056 : ffff994207e8e468) 00000100:00000001:1.0:1775768090.292354:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000001:3.0:1775768090.292358:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000020:00000001:3.0:1775768090.292362:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768090.292363:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fd880 x1862022294272768/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768115 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:3.0:1775768090.292370:0:13471:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768090.292381:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768090.292391:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00020000:00000001:3.0:1775768090.292397:0:13471:0:(lov_io.c:1108:lov_io_lock()) Process entered 00020000:00000001:3.0:1775768090.292404:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00100000:1.0:1775768090.292405:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294272768:192.168.202.156@tcp:38:kworker.0 00000020:00000001:3.0:1775768090.292410:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:3.0:1775768090.292414:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:3.0:1775768090.292419:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768090.292420:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1775768090.292426:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768090.292430:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ff480 x1862022294272896/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:3.0:1775768090.292431:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768090.292439:0:13471:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768090.292449:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:3.0:1775768090.292453:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768090.292462:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768090.292465:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000001:3.0:1775768090.292465:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00000100:00000001:1.0:1775768090.292467:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768090.292468:0:13471:0:(lov_io.c:1114:lov_io_start()) Process entered 00020000:00000001:3.0:1775768090.292472:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775768090.292474:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 02000000:00000001:1.0:1775768090.292475:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000001:3.0:1775768090.292478:0:13471:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 02000000:00000001:1.0:1775768090.292480:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:3.0:1775768090.292480:0:13471:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 02000000:00000001:1.0:1775768090.292487:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1775768090.292487:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:3.0:1775768090.292494:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:3.0:1775768090.292499:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768090.292500:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294272896:192.168.202.156@tcp:38:kworker.0 00000008:00000001:3.0:1775768090.292508:0:13471:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000100:00000001:1.0:1775768090.292511:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000008:00000001:3.0:1775768090.292512:0:13471:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000008:00000001:3.0:1775768090.292514:0:13471:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107754938304 : -112965954613312 : ffff994210674fc0) 02000000:00000001:1.0:1775768090.292515:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768090.292518:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000020:3.0:1775768090.292521:0:13471:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994210674fc0 ready 0|-|- wr 1|-|+ rd 0|- 9 in flight 02000000:00000001:1.0:1775768090.292522:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768090.292528:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543fc00. 00000008:00000001:3.0:1775768090.292528:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:3.0:1775768090.292532:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 02000000:00000001:1.0:1775768090.292533:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1775768090.292535:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1775768090.292544:0:13471:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000400:00000200:1.0:1775768090.292549:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00000020:3.0:1775768090.292549:0:13471:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994218232380 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000400:00000010:1.0:1775768090.292558:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000008:00000001:3.0:1775768090.292561:0:13471:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:1.0:1775768090.292563:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000008:00000001:3.0:1775768090.292564:0:13471:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768090.292573:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775768090.292575:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294272896, portal 10 00020000:00000001:3.0:1775768090.292576:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768090.292584:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00020000:00000001:3.0:1775768090.292584:0:13471:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768090.292587:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641209984 : -112966068341632 : ffff9942099ff480) 00000020:00000001:3.0:1775768090.292588:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768090.292594:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00020000:00000001:3.0:1775768090.292595:0:13471:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00020000:00000001:3.0:1775768090.292597:0:13471:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000100:00000040:1.0:1775768090.292598:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ff480 x1862022294272896/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768115 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775768090.292600:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000001:3.0:1775768090.292605:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00020000:00000001:3.0:1775768090.292610:0:13471:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768090.292621:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00020000:00000001:3.0:1775768090.292622:0:13471:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000020:00000001:3.0:1775768090.292629:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00000100:00000040:1.0:1775768090.292633:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000020:00000001:3.0:1775768090.292634:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000400:00000010:1.0:1775768090.292636:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00020000:00000001:3.0:1775768090.292637:0:13471:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00000100:00000200:1.0:1775768090.292639:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294272896, offset 0 00020000:00000001:3.0:1775768090.292640:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775768090.292646:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000400:00000200:1.0:1775768090.292650:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000020:00000001:3.0:1775768090.292650:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:3.0:1775768090.292656:0:13471:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768090.292662:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:3.0:1775768090.292664:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768090.292667:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00020000:00000001:3.0:1775768090.292674:0:13471:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000400:00000200:1.0:1775768090.292676:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:3.0:1775768090.292680:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000400:00000200:1.0:1775768090.292682:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000020:00000001:3.0:1775768090.292684:0:13471:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000400:00000200:1.0:1775768090.292686:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000020:00000001:3.0:1775768090.292687:0:13471:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000400:00000200:1.0:1775768090.292691:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000020:00000001:3.0:1775768090.292692:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:3.0:1775768090.292697:0:13471:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00000400:00000200:1.0:1775768090.292698:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225305:225305:256:4294967295] 192.168.202.156@tcp LPNI seq info [225301:225301:8:4294967295] 00020000:00000001:3.0:1775768090.292701:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775768090.292706:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000400:00000200:1.0:1775768090.292707:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:3.0:1775768090.292709:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00020000:00000001:3.0:1775768090.292712:0:13471:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768090.292716:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768090.292723:0:13471:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000020:00000001:3.0:1775768090.292727:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000400:00000200:1.0:1775768090.292730:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000020:00000001:3.0:1775768090.292730:0:13471:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768090.292737:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:3.0:1775768090.292741:0:13471:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:3.0:1775768090.292743:0:13471:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:3.0:1775768090.292744:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:3.0:1775768090.292748:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:3.0:1775768090.292749:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff99420693ecf0 00000800:00000200:1.0:1775768090.292751:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000020:00000010:3.0:1775768090.292755:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210dcea00. 00000800:00000010:1.0:1775768090.292758:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000020:00000010:3.0:1775768090.292759:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942079817e0. 00000008:00000010:3.0:1775768090.292767:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994208b51650. 00020000:00000010:3.0:1775768090.292776:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994218000000. 00000800:00000200:1.0:1775768090.292777:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768090.292785:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00008000:00000010:3.0:1775768090.292785:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994207e0ade8. 00000080:00000010:3.0:1775768090.292799:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99420541b0a8. 00000800:00000200:1.1:1775768090.292800:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000080:00000010:3.0:1775768090.292806:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077c1da8. 00000800:00000200:1.1:1775768090.292806:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000020:00000010:3.0:1775768090.292814:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcf400. 00000800:00000100:1.0:1775768090.292821:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000008:00000010:3.0:1775768090.292822:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942086f8160. 00000800:00000010:1.0:1775768090.292826:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00020000:00000010:3.0:1775768090.292828:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994207e8ed00. 00000400:00000200:1.0:1775768090.292832:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00008000:00000010:3.0:1775768090.292834:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994209270ef0. 00000080:00000010:3.0:1775768090.292842:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942090e3960. 00000400:00000200:1.0:1775768090.292843:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000020:00000010:3.0:1775768090.292851:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dce000. 00000400:00000200:1.0:1775768090.292852:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768090.292856:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000020:00000010:3.0:1775768090.292857:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff99420693ecf0. 00000100:00000001:1.0:1775768090.292860:0:2399:0:(events.c:34:request_out_callback()) Process entered 00020000:00000001:3.0:1775768090.292864:0:13471:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00000100:00000200:1.0:1775768090.292866:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ff480 x1862022294272896/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768115 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:3.0:1775768090.292869:0:13471:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:3.0:1775768090.292872:0:13471:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000080:00200000:3.0:1775768090.292878:0:13471:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017f:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00000001:1.0:1775768090.292880:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768090.292884:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ff480 x1862022294272896/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768115 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000001:3.0:1775768090.292887:0:13471:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:3.0:1775768090.292892:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:3.0:1775768090.292899:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff99420693e000 00000100:00000001:1.0:1775768090.292900:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768090.292908:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000010:3.0:1775768090.292909:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210dcf000. 00000100:00000001:1.0:1775768090.292912:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1775768090.292916:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207981de0. 00000100:00000040:1.0:1775768090.292917:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:1.0:1775768090.292927:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00000010:3.0:1775768090.292927:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994208b50000. 00000100:00000001:1.0:1775768090.292934:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300015117 00000100:00000001:1.0:1775768090.292937:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300016446 00020000:00000010:3.0:1775768090.292937:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994218000300. 00008000:00000010:3.0:1775768090.292944:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994207e08e20. 00000100:00000001:1.0:1775768090.292945:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300015117 00000100:00000001:1.0:1775768090.292952:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300016446 00000080:00000010:3.0:1775768090.292952:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99420541b1f8. 00000100:00000001:1.0:1775768090.292959:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000010:3.0:1775768090.292960:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077c0248. 00000100:00000001:1.0:1775768090.292966:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1775768090.292967:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcf200. 00000100:00000001:1.0:1775768090.292976:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000008:00000010:3.0:1775768090.292977:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942086f8420. 00020000:00000010:3.0:1775768090.292981:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994207e8e400. 00000100:00100000:1.0:1775768090.292984:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768090/real 1775768090] req@ffff9942099ff480 x1862022294272896/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00008000:00000010:3.0:1775768090.292989:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994209270a90. 00000080:00000010:3.0:1775768090.292995:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942090e33c0. 00000020:00000010:3.0:1775768090.293005:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcf800. 00000400:00000001:1.0:1775768090.293008:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000010:3.0:1775768090.293011:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff99420693e000. 00000400:00000001:1.0:1775768090.293012:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000080:00000001:3.0:1775768090.293016:0:13471:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:1.0:1775768090.293017:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000080:00000001:3.0:1775768090.293038:0:13471:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768090.293041:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768090.293045:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768090.293049:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768090.293055:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ff480 x1862022294272896/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768090.293064:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ff480 x1862022294272896/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768090.293072:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768090.293074:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768090.293076:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768090.293078:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768090.293082:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ff480 x1862022294272896/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768090.293088:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768090.293092:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ff480 x1862022294272896/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768115 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768090.293098:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768090.293102:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768090.293103:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768090.293104:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768090.293106:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768090.293109:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768090.293112:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768090.293115:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768090.293117:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768090.293120:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ff480 x1862022294272896/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768115 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768090.293128:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294272896:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768090.293131:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768090.293133:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768090.293136:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fd880 x1862022294272768/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768115 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768090.293143:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768090.293145:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768090.293147:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543dc00. 02000000:00000001:1.0:1775768090.293149:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768090.293151:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768090.293152:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:1.0:1775768090.293174:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768090.293177:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029400. 02000000:00000001:1.0:1775768090.293180:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768090.293182:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768090.293183:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fd880. 00000100:00000001:1.0:1775768090.293185:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768090.293186:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768090.293187:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768090.293190:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ff480 x1862022294272896/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768115 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768090.293199:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768090.293200:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768090.293201:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543fc00. 02000000:00000001:1.0:1775768090.293202:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768090.293203:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768090.293204:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:1.0:1775768090.293205:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768090.293206:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002bc00. 02000000:00000001:1.0:1775768090.293207:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768090.293208:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768090.293209:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ff480. 00000100:00000001:1.0:1775768090.293210:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768090.293211:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768090.293212:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768090.293215:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768090.293216:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768090.293217:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768090.293219:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768090.293220:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768090.293222:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fd180 x1862022294273024/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768090.293227:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768090.293228:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768090.293230:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768090.293231:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768090.293233:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768090.293234:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294273024:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768090.293237:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768090.293238:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768090.293239:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768090.293240:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768090.293242:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543fc00. 02000000:00000001:1.0:1775768090.293244:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768090.293247:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768090.293249:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000400:00000010:1.0:1775768090.293252:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000200:1.0:1775768090.293254:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294273024, portal 4 00000100:00000001:1.0:1775768090.293255:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768090.293256:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641201024 : -112966068350592 : ffff9942099fd180) 00000100:00000040:1.0:1775768090.293260:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fd180 x1862022294273024/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768115 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768090.293264:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768090.293266:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768090.293267:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768090.293269:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294273024, offset 0 00000400:00000200:1.0:1775768090.293272:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768090.293275:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768090.293277:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768090.293279:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768090.293281:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768090.293283:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768090.293284:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768090.293287:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225306:225306:256:4294967295] 192.168.202.156@tcp LPNI seq info [225302:225302:8:4294967295] 00000400:00000200:1.0:1775768090.293290:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768090.293294:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768090.293297:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768090.293299:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000800:00000200:1.0:1775768090.293302:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768090.293304:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775768090.293307:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768090.293309:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000100:1.0:1775768090.293312:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768090.293313:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768090.293315:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768090.293317:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768090.293319:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768090.293321:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768090.293323:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768090.293325:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fd180 x1862022294273024/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768115 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768090.293330:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768090.293332:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fd180 x1862022294273024/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768115 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768090.293336:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768090.293337:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768090.293339:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768090.293341:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:1.0:1775768090.293343:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768090.293345:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300015141 00000100:00000001:1.0:1775768090.293346:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300016446 00000100:00000001:1.0:1775768090.293348:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300015141 00000100:00000001:1.0:1775768090.293349:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300016446 00000100:00000001:1.0:1775768090.293351:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768090.293352:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768090.293353:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768090.293355:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768090/real 1775768090] req@ffff9942099fd180 x1862022294273024/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768090.293360:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768090.293361:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768090.293362:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768090.293363:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768090.293364:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768090.293365:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768090.293367:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fd180 x1862022294273024/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768090.293372:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fd180 x1862022294273024/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768090.293376:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768090.293378:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768090.293379:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768090.293380:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768090.293382:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fd180 x1862022294273024/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768090.293387:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768090.293391:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fd180 x1862022294273024/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768115 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768090.293398:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768090.293400:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768090.293402:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768090.293403:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768090.293405:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768090.293409:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768090.293411:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768090.293415:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768090.293417:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768090.293422:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fd180 x1862022294273024/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768115 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768090.293430:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294273024:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768090.293434:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768090.293437:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fed80 x1862022294273152/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768090.293442:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768090.293443:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768090.293446:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768090.293447:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768090.293449:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768090.293451:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294273152:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768090.293454:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768090.293456:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768090.293457:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768090.293459:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768090.293462:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543dc00. 02000000:00000001:1.0:1775768090.293464:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768090.293467:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768090.293470:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000400:00000010:1.0:1775768090.293472:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000200:1.0:1775768090.293475:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294273152, portal 4 00000100:00000001:1.0:1775768090.293477:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768090.293478:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641208192 : -112966068343424 : ffff9942099fed80) 00000100:00000040:1.0:1775768090.293484:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fed80 x1862022294273152/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768115 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768090.293490:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768090.293492:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768090.293494:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768090.293497:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294273152, offset 0 00000400:00000200:1.0:1775768090.293501:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768090.293505:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768090.293509:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768090.293512:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768090.293514:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768090.293516:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768090.293517:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768090.293520:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225307:225307:256:4294967295] 192.168.202.156@tcp LPNI seq info [225303:225303:8:4294967295] 00000400:00000200:1.0:1775768090.293523:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768090.293527:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768090.293531:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768090.293533:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000800:00000200:1.0:1775768090.293536:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768090.293538:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775768090.293541:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768090.293543:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000100:1.0:1775768090.293545:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768090.293547:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768090.293549:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768090.293551:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768090.293553:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768090.293554:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768090.293556:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768090.293558:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fed80 x1862022294273152/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768115 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768090.293564:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768090.293565:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fed80 x1862022294273152/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768115 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768090.293570:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768090.293571:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768090.293573:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768090.293575:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:1.0:1775768090.293577:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768090.293578:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300015144 00000100:00000001:1.0:1775768090.293580:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300016446 00000100:00000001:1.0:1775768090.293582:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300015144 00000100:00000001:1.0:1775768090.293583:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300016446 00000100:00000001:1.0:1775768090.293585:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768090.293586:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768090.293587:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768090.293589:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768090/real 1775768090] req@ffff9942099fed80 x1862022294273152/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768090.293595:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768090.293596:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768090.293597:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768090.293599:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768090.293600:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768090.293601:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768090.293602:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fed80 x1862022294273152/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768090.293608:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fed80 x1862022294273152/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768090.293612:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768090.293613:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768090.293614:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768090.293615:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768090.293617:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fed80 x1862022294273152/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768090.293620:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768090.293623:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fed80 x1862022294273152/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768115 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768090.293627:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768090.293628:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768090.293629:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768090.293630:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768090.293631:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768090.293633:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768090.293635:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768090.293637:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768090.293638:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768090.293641:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fed80 x1862022294273152/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768115 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768090.293646:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294273152:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768090.293648:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768090.293649:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768090.293651:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fd180 x1862022294273024/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768115 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768090.293655:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768090.293656:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768090.293657:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543fc00. 02000000:00000001:1.0:1775768090.293658:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768090.293659:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768090.293660:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:1.0:1775768090.293661:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768090.293662:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002a800. 02000000:00000001:1.0:1775768090.293664:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768090.293664:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768090.293665:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fd180. 00000100:00000001:1.0:1775768090.293667:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768090.293668:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768090.293669:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768090.293671:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fed80 x1862022294273152/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768115 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768090.293674:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768090.293675:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768090.293676:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543dc00. 02000000:00000001:1.0:1775768090.293677:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768090.293678:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768090.293678:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:1.0:1775768090.293679:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768090.293680:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029800. 02000000:00000001:1.0:1775768090.293681:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768090.293682:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768090.293683:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fed80. 00000100:00000001:1.0:1775768090.293685:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768090.293686:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768090.293687:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768090.293689:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768090.293690:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768090.293691:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768090.293692:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768090.353285:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768090.353290:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768090.353296:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768090.353297:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768090.353301:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768090.353303:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768090.353306:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768090.353308:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768090.353311:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768090.353312:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768090.353357:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768090.353362:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768090.353366:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768090.417243:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768090.417250:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768090.417252:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768090.417258:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768090.417262:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768090.417266:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768090.417269:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768090.417272:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768090.417274:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768091.313242:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768091.313244:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768091.313245:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768091.313249:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768091.313249:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768091.313250:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768091.313253:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768091.313253:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768091.313253:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768091.313260:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775768091.313261:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775768091.313261:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768091.313266:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768091.313268:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768091.313268:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768091.313271:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768091.313271:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768091.313272:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768091.313275:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768091.313278:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768091.313280:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768091.313283:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768091.313286:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768091.313288:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768091.313288:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768091.313291:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768091.313294:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768091.377278:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768091.377285:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768091.377292:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768091.377293:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768091.377297:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768091.377298:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768091.377302:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768091.377303:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768091.377306:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768091.377308:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768091.377337:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768091.377339:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768091.377342:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768091.441273:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768091.441279:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768091.441280:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768091.441285:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768091.441288:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768091.441292:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768091.441294:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768091.441296:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768091.441298:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768091.633260:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4300016446, at 4300016448 00000100:00000010:1.0:1775768091.633272:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cba000. 00000100:00000040:0.0:1775768091.633345:0:14953:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4300016446, at 4300016448 00000100:00000010:0.0:1775768091.633349:0:14953:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cba360. 00000100:00000001:1.0:1775768092.337242:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768092.337242:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768092.337248:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768092.337249:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768092.337252:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768092.337252:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768092.337259:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:1.0:1775768092.337260:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768092.337266:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768092.337267:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768092.337271:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768092.337271:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768092.337271:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768092.337275:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768092.337276:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768092.337280:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768092.337280:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768092.337285:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768092.337286:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768092.337286:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768092.337290:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768092.337291:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768092.337294:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768092.337294:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768092.337300:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768092.337302:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768092.337304:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768092.401227:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768092.401233:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768092.401235:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768092.401237:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768092.401241:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768092.401249:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768092.401251:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768092.401252:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768092.401255:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768092.401256:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768092.401258:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768092.401259:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768092.401261:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768092.401262:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768092.401264:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768092.401265:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768092.401308:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768092.401312:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768092.401316:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768092.465217:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768092.465223:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768092.465225:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768092.465229:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768092.465232:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768092.465235:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768092.465237:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768092.465239:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768092.465241:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768093.361239:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768093.361245:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768093.361247:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768093.361253:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768093.361257:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768093.361257:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768093.361261:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768093.361262:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768093.361264:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768093.361265:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768093.361267:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768093.361271:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768093.361272:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768093.361277:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768093.361281:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768093.361281:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768093.361287:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768093.361287:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768093.361290:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768093.361291:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768093.361297:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768093.361297:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768093.361303:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768093.361307:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768093.361310:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768093.361312:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768093.361315:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768093.425263:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768093.425282:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768093.425288:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768093.425289:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768093.425293:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768093.425295:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768093.425298:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768093.425300:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768093.425303:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768093.425305:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768093.425353:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768093.425356:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768093.425359:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768093.489280:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768093.489295:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768093.489297:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768093.489307:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768093.489311:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768093.489315:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768093.489318:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768093.489321:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768093.489324:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768094.385254:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768094.385254:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768094.385256:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768094.385261:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768094.385261:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768094.385262:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768094.385263:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768094.385263:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768094.385266:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768094.385270:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775768094.385270:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775768094.385271:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768094.385276:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768094.385277:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768094.385277:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768094.385281:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768094.385282:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768094.385282:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768094.385285:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768094.385289:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768094.385291:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768094.385294:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768094.385296:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768094.385298:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768094.385299:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768094.385302:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768094.385304:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768094.449266:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768094.449272:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768094.449278:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768094.449280:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768094.449284:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768094.449286:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768094.449289:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768094.449290:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768094.449293:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768094.449295:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768094.449339:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768094.449342:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768094.449344:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768094.513241:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768094.513246:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768094.513248:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768094.513253:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768094.513256:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768094.513259:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768094.513261:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768094.513263:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768094.513265:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768095.409261:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775768095.409264:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768095.409265:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768095.409270:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:0.0:1775768095.409271:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775768095.409272:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768095.409272:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768095.409276:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1775768095.409276:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768095.409277:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00080000:0.0:1775768095.409280:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768095.409283:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:2.0:1775768095.409284:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768095.409285:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775768095.409287:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768095.409291:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1775768095.409292:0:13471:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5347 00000100:00000001:1.0:1775768095.409292:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768095.409296:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768095.409297:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768095.409301:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768095.409302:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000400:00000200:0.0:1775768095.409304:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768095.409307:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768095.409311:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768095.409312:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768095.409316:0:13471:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768095.409320:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768095.409323:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768095.409325:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768095.409327:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768095.409330:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768095.409331:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:0.0:1775768095.409333:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768095.409335:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:0.0:1775768095.409337:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:0.0:1775768095.409340:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:0.0:1775768095.409342:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768095.409344:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768095.409345:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768095.409346:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768095.409348:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:0.0:1775768095.409351:0:13471:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768095.409352:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768095.409354:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768095.409356:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768095.409358:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768095.409359:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775768095.409361:0:13471:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768095.409369:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fc380. 00000020:00000040:0.0:1775768095.409372:0:13471:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:0.0:1775768095.409375:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768095.409376:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768095.409379:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768095.409381:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768095.409385:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768095.409392:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002b800. 00000100:00000001:0.0:1775768095.409397:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768095.409404:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fc380 x1862022294273280/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768095.409415:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768095.409420:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768095.409426:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002957419c] to pc [ptlrpcd_rcv+-1] req@ffff9942099fc380 x1862022294273280/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768095.409454:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768095.409456:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768095.409459:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:0.0:1775768095.409461:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775768095.409461:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768095.409464:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768095.409468:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1775768095.409470:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775768095.409471:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768095.409475:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775768095.409475:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768095.409479:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1775768095.409480:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768095.409481:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768095.409487:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768095.409491:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5347 00000400:00000200:0.0:1775768095.409496:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768095.409501:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768095.409503:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768095.409505:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768095.409507:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768095.409508:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768095.409510:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768095.409511:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768095.409512:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:0.0:1775768095.409513:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768095.409515:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:0.0:1775768095.409516:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:0.0:1775768095.409517:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:0.0:1775768095.409519:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768095.409521:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768095.409523:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768095.409524:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768095.409526:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00080000:0.0:1775768095.409528:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768095.409530:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768095.409532:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768095.409537:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768095.409539:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768095.409541:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775768095.409542:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775768095.409547:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000008:00100000:0.0:1775768095.409549:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1775768095.409551:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000008:00000001:0.0:1775768095.409554:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768095.409558:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768095.409563:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fc380 x1862022294273280/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1775768095.409564:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fca80. 00000020:00000040:0.0:1775768095.409569:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:0.0:1775768095.409576:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768095.409580:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768095.409583:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768095.409585:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768095.409589:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768095.409592:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768095.409593:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768095.409597:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775768095.409597:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768095.409601:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768095.409605:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294273280:192.168.202.156@tcp:250:kworker.0 02000000:00000010:0.0:1775768095.409608:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220028400. 00000100:00000001:2.0:1775768095.409610:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1775768095.409613:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768095.409616:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:0.0:1775768095.409621:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fca80 x1862022294273408/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1775768095.409621:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768095.409628:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1775768095.409632:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768095.409634:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775768095.409636:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f800. 00000100:00000040:0.0:1775768095.409637:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000046a3ca2f] to pc [ptlrpcd_rcv+-1] req@ffff9942099fca80 x1862022294273408/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1775768095.409641:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768095.409650:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768095.409655:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:2.0:1775768095.409655:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00080000:0.0:1775768095.409661:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775768095.409662:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000400:00000010:2.0:1775768095.409663:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000100:00080000:0.0:1775768095.409664:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768095.409669:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768095.409672:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5347 00000400:00000010:2.0:1775768095.409675:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000400:00000200:0.0:1775768095.409676:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768095.409681:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768095.409683:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768095.409686:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000200:2.0:1775768095.409686:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294273280, portal 25 00000100:00000001:0.0:1775768095.409690:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775768095.409692:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000040:0.0:1775768095.409693:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768095.409696:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:2.0:1775768095.409696:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641197440 : -112966068354176 : ffff9942099fc380) 00000020:00000001:0.0:1775768095.409701:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768095.409703:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:0.0:1775768095.409704:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768095.409706:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:0.0:1775768095.409709:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:2.0:1775768095.409710:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fc380 x1862022294273280/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768120 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775768095.409711:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:0.0:1775768095.409717:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768095.409720:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768095.409723:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768095.409726:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768095.409730:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:0.0:1775768095.409736:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775768095.409739:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1775768095.409740:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768095.409745:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:2.0:1775768095.409746:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 02000000:00000001:0.0:1775768095.409749:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:2.0:1775768095.409750:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f880. 02000000:00000001:0.0:1775768095.409755:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1775768095.409756:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294273280, offset 0 00000100:00000001:0.0:1775768095.409758:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775768095.409762:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775768095.409767:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768095.409768:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00000001:0.0:1775768095.409772:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768095.409778:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fe680. 00000400:00000200:2.0:1775768095.409780:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000040:0.0:1775768095.409784:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:0.0:1775768095.409788:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000200:2.0:1775768095.409791:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 02000000:00000001:0.0:1775768095.409792:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768095.409797:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768095.409800:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:2.0:1775768095.409802:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768095.409807:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768095.409812:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 02000000:00000001:0.0:1775768095.409813:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768095.409816:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 02000000:00000010:0.0:1775768095.409820:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220029800. 00000400:00000200:2.0:1775768095.409824:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225308:225308:256:4294967295] 192.168.202.156@tcp LPNI seq info [225304:225304:8:4294967295] 00000100:00000001:0.0:1775768095.409827:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768095.409832:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00100000:0.0:1775768095.409836:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fe680 x1862022294273536/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768095.409847:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768095.409851:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768095.409854:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000100:00000040:0.0:1775768095.409858:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ba41ba08] to pc [ptlrpcd_rcv+-1] req@ffff9942099fe680 x1862022294273536/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768095.409869:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1775768095.409869:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00080000:0.0:1775768095.409874:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768095.409876:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775768095.409877:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000800:00000010:2.0:1775768095.409878:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3a00. 00000100:00080000:0.0:1775768095.409879:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768095.409885:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768095.409887:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5347 00000400:00000200:0.0:1775768095.409893:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000800:00000200:2.0:1775768095.409894:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000400:00000200:0.0:1775768095.409898:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768095.409901:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000800:00000200:2.0:1775768095.409902:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000100:00000040:0.0:1775768095.409904:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768095.409911:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768095.409917:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000800:00000200:2.1:1775768095.409917:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:0.0:1775768095.409922:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768095.409926:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000800:00000200:2.1:1775768095.409926:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000020:00000040:0.0:1775768095.409930:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:0.0:1775768095.409933:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768095.409938:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000800:00000100:2.0:1775768095.409938:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000020:00000001:0.0:1775768095.409943:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000800:00000010:2.0:1775768095.409944:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3a00. 00000020:00000001:0.0:1775768095.409950:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000400:00000200:2.0:1775768095.409953:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:0.0:1775768095.409956:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768095.409960:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1775768095.409965:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000100:00000040:0.0:1775768095.409966:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000200:2.0:1775768095.409987:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f880 00000400:00000010:2.0:1775768095.409992:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f880. 00000100:00000001:0.0:1775768095.409993:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768095.409997:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:0.0:1775768095.409999:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768095.410002:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768095.410004:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768095.410006:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768095.410008:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768095.410008:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1775768095.410010:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775768095.410014:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:0.0:1775768095.410016:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768095.410019:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1775768095.410019:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fc380 x1862022294273280/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768120 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1775768095.410022:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fdc00. 00000020:00000040:0.0:1775768095.410026:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:0.0:1775768095.410031:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768095.410033:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768095.410038:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768095.410040:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768095.410043:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768095.410044:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 02000000:00000010:0.0:1775768095.410047:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002a800. 00000100:00000040:2.0:1775768095.410052:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fc380 x1862022294273280/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768120 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768095.410053:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768095.410062:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fdc00 x1862022294273664/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775768095.410072:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768095.410075:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775768095.410076:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768095.410079:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768095.410084:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001372d9ae] to pc [ptlrpcd_rcv+-1] req@ffff9942099fdc00 x1862022294273664/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768095.410086:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768095.410095:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768095.410096:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00080000:0.0:1775768095.410098:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775768095.410101:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4300021565 00000100:00080000:0.0:1775768095.410102:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775768095.410104:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768095.410107:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768095.410109:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768095.410111:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5347 00000100:00000010:2.0:1775768095.410111:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d6300. 00000400:00000200:0.0:1775768095.410116:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768095.410119:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768095.410120:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1775768095.410122:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000040:0.0:1775768095.410123:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768095.410127:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768095.410129:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768095.410131:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768095.410133:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:2.0:1775768095.410133:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4300021565 00000020:00000040:0.0:1775768095.410135:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:0.0:1775768095.410139:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768095.410140:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000100:00000010:2.0:1775768095.410140:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d6480. 00000020:00000001:0.0:1775768095.410142:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:0.0:1775768095.410146:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000001:2.0:1775768095.410147:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000040:0.0:1775768095.410148:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768095.410174:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768095.410176:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768095.410177:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768095.410180:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:0.0:1775768095.410182:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768095.410184:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768095.410186:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1775768095.410186:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768095.410190:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768095.410192:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768095.410193:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768095.410197:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775768095.410198:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000008:00100000:0.0:1775768095.410199:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:0.0:1775768095.410202:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768095.410204:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768095.410208:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fce00. 00000100:00100000:2.0:1775768095.410208:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768095/real 1775768095] req@ffff9942099fc380 x1862022294273280/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1775768095.410212:0:13471:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:0.0:1775768095.410217:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768095.410219:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768095.410223:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768095.410226:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768095.410228:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768095.410234:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002bc00. 00000400:00000001:2.0:1775768095.410236:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1775768095.410237:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1775768095.410239:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768095.410242:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000100:00100000:0.0:1775768095.410243:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fce00 x1862022294273792/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1775768095.410248:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768095.410253:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:0.0:1775768095.410256:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768095.410259:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768095.410260:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:0.0:1775768095.410264:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000060e5e9d1] to pc [ptlrpcd_rcv+-1] req@ffff9942099fce00 x1862022294273792/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768095.410269:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fc380 x1862022294273280/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768095.410274:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768095.410280:0:13471:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5357) 00000100:00000200:2.0:1775768095.410289:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fc380 x1862022294273280/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768095.410298:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768095.410300:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768095.410303:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768095.410305:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768095.410309:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fc380 x1862022294273280/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768095.410316:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768095.410321:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fc380 x1862022294273280/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768120 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768095.410329:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768095.410332:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768095.410334:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768095.410335:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768095.410338:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768095.410340:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768095.410343:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768095.410347:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768095.410350:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768095.410355:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fc380 x1862022294273280/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768120 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768095.410365:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294273280:192.168.202.156@tcp:250:kworker.0 00000100:00000001:2.0:1775768095.410369:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768095.410371:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768095.410374:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fc380 x1862022294273280/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768120 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768095.410381:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768095.410384:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768095.410387:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f800. 02000000:00000001:2.0:1775768095.410391:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768095.410393:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768095.410395:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:2.0:1775768095.410397:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768095.410400:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002b800. 02000000:00000001:2.0:1775768095.410404:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768095.410407:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768095.410408:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fc380. 00000100:00000001:2.0:1775768095.410411:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768095.410413:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768095.410415:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768095.410423:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768095.410425:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768095.410427:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768095.410429:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768095.410430:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768095.410434:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fca80 x1862022294273408/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768095.410442:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768095.410444:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768095.410446:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768095.410448:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768095.410451:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768095.410454:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294273408:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768095.410458:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768095.410460:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768095.410462:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768095.410464:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768095.410467:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f800. 02000000:00000001:2.0:1775768095.410469:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768095.410473:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768095.410476:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768095.410480:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768095.410483:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294273408, portal 10 00000100:00000001:2.0:1775768095.410486:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768095.410487:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641199232 : -112966068352384 : ffff9942099fca80) 00000100:00000040:2.0:1775768095.410494:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fca80 x1862022294273408/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768120 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768095.410502:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768095.410505:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768095.410507:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f880. 00000100:00000200:2.0:1775768095.410510:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294273408, offset 0 00000400:00000200:2.0:1775768095.410514:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768095.410519:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768095.410523:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768095.410526:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768095.410529:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768095.410532:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768095.410535:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768095.410539:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225309:225309:256:4294967295] 192.168.202.156@tcp LPNI seq info [225305:225305:8:4294967295] 00000400:00000200:2.0:1775768095.410543:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768095.410551:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768095.410556:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768095.410559:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3a00. 00000800:00000200:2.0:1775768095.410563:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768095.410566:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775768095.410571:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768095.410574:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1775768095.410578:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768095.410580:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3a00. 00000400:00000200:2.0:1775768095.410583:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768095.410587:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768095.410591:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f880 00000400:00000010:2.0:1775768095.410593:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f880. 00000100:00000001:2.0:1775768095.410595:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768095.410600:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fca80 x1862022294273408/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768120 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768095.410615:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768095.410618:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fca80 x1862022294273408/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768120 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768095.410626:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768095.410627:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768095.410630:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768095.410635:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:2.0:1775768095.410638:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768095.410641:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300020243 00000100:00000001:2.0:1775768095.410644:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300021565 00000100:00000001:2.0:1775768095.410646:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300020243 00000100:00000001:2.0:1775768095.410648:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300021565 00000100:00000001:2.0:1775768095.410650:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768095.410653:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768095.410654:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768095.410658:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768095/real 1775768095] req@ffff9942099fca80 x1862022294273408/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768095.410667:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768095.410668:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768095.410670:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768095.410672:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768095.410674:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768095.410677:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768095.410681:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fca80 x1862022294273408/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768095.410690:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fca80 x1862022294273408/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768095.410698:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768095.410700:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768095.410702:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768095.410703:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768095.410707:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fca80 x1862022294273408/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768095.410714:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768095.410718:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fca80 x1862022294273408/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768120 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768095.410725:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768095.410728:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768095.410730:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768095.410731:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768095.410733:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768095.410736:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768095.410739:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768095.410743:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768095.410745:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768095.410750:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fca80 x1862022294273408/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768120 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768095.410759:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294273408:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768095.410763:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768095.410766:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fe680 x1862022294273536/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768095.410773:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768095.410775:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768095.410777:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768095.410779:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768095.410781:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768095.410785:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294273536:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768095.410789:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768095.410791:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768095.410793:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768095.410795:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768095.410798:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1d000. 02000000:00000001:2.0:1775768095.410800:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768095.410803:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768095.410806:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768095.410809:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768095.410813:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294273536, portal 10 00000100:00000001:2.0:1775768095.410815:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768095.410817:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641206400 : -112966068345216 : ffff9942099fe680) 00000100:00000040:2.0:1775768095.410823:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fe680 x1862022294273536/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768120 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768095.410830:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768095.410832:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768095.410835:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f880. 00000100:00000200:2.0:1775768095.410837:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294273536, offset 0 00000400:00000200:2.0:1775768095.410841:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768095.410846:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768095.410850:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768095.410853:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768095.410856:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768095.410859:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768095.410862:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768095.410865:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225310:225310:256:4294967295] 192.168.202.156@tcp LPNI seq info [225306:225306:8:4294967295] 00000400:00000200:2.0:1775768095.410870:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768095.410877:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768095.410882:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768095.410886:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3a00. 00000800:00000200:2.0:1775768095.410890:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768095.410893:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775768095.410898:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768095.410901:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1775768095.410905:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768095.410907:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3a00. 00000400:00000200:2.0:1775768095.410911:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768095.410915:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768095.410918:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f880 00000400:00000010:2.0:1775768095.410920:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f880. 00000100:00000001:2.0:1775768095.410922:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768095.410926:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fe680 x1862022294273536/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768120 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768095.410935:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768095.410939:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fe680 x1862022294273536/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768120 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768095.410946:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768095.410948:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768095.410951:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768095.410954:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:2.0:1775768095.410958:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768095.410960:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300020235 00000100:00000001:2.0:1775768095.410963:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300021565 00000100:00000001:2.0:1775768095.410966:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300020235 00000100:00000001:2.0:1775768095.410968:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300021565 00000100:00000001:2.0:1775768095.410983:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768095.410985:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768095.410987:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768095.410991:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768095/real 1775768095] req@ffff9942099fe680 x1862022294273536/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768095.411001:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768095.411003:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768095.411005:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768095.411008:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768095.411010:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768095.411012:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768095.411016:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fe680 x1862022294273536/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768095.411026:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fe680 x1862022294273536/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768095.411033:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768095.411035:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768095.411037:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768095.411038:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768095.411042:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fe680 x1862022294273536/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768095.411050:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768095.411054:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fe680 x1862022294273536/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768120 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768095.411063:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768095.411065:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768095.411067:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768095.411069:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768095.411071:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768095.411074:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768095.411076:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768095.411080:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768095.411083:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768095.411087:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fe680 x1862022294273536/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768120 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768095.411103:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294273536:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768095.411108:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768095.411112:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fdc00 x1862022294273664/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768095.411120:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768095.411121:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768095.411123:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768095.411126:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768095.411128:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768095.411131:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294273664:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768095.411135:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768095.411138:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768095.411140:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768095.411141:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768095.411145:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1e400. 02000000:00000001:2.0:1775768095.411147:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768095.411151:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768095.411174:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768095.411178:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768095.411181:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294273664, portal 4 00000100:00000001:2.0:1775768095.411184:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768095.411186:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641203712 : -112966068347904 : ffff9942099fdc00) 00000100:00000040:2.0:1775768095.411192:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fdc00 x1862022294273664/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768120 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768095.411199:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768095.411202:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768095.411204:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f880. 00000100:00000200:2.0:1775768095.411207:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294273664, offset 0 00000400:00000200:2.0:1775768095.411211:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768095.411216:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768095.411220:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768095.411224:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768095.411227:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768095.411230:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768095.411233:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768095.411236:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225311:225311:256:4294967295] 192.168.202.156@tcp LPNI seq info [225307:225307:8:4294967295] 00000400:00000200:2.0:1775768095.411241:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768095.411248:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768095.411254:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768095.411257:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3a00. 00000800:00000200:2.0:1775768095.411261:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768095.411265:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775768095.411269:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768095.411272:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1775768095.411277:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768095.411279:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3a00. 00000400:00000200:2.0:1775768095.411282:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768095.411287:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768095.411290:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f880 00000400:00000010:2.0:1775768095.411292:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f880. 00000100:00000001:2.0:1775768095.411295:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768095.411298:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fdc00 x1862022294273664/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768120 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768095.411307:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768095.411311:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fdc00 x1862022294273664/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768120 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768095.411318:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768095.411320:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768095.411323:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768095.411326:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:2.0:1775768095.411329:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768095.411332:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300020259 00000100:00000001:2.0:1775768095.411335:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300021565 00000100:00000001:2.0:1775768095.411338:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300020259 00000100:00000001:2.0:1775768095.411340:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300021565 00000100:00000001:2.0:1775768095.411342:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768095.411345:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768095.411347:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768095.411351:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768095/real 1775768095] req@ffff9942099fdc00 x1862022294273664/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768095.411359:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768095.411361:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768095.411363:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768095.411365:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768095.411367:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768095.411370:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768095.411373:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fdc00 x1862022294273664/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768095.411383:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fdc00 x1862022294273664/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768095.411391:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768095.411392:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768095.411395:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768095.411396:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768095.411400:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fdc00 x1862022294273664/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768095.411408:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768095.411412:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fdc00 x1862022294273664/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768120 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768095.411419:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768095.411422:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768095.411424:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768095.411425:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768095.411427:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768095.411431:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768095.411434:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768095.411438:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768095.411440:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768095.411445:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fdc00 x1862022294273664/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768120 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768095.411455:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294273664:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768095.411459:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768095.411462:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fce00 x1862022294273792/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768095.411470:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768095.411472:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768095.411474:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768095.411477:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768095.411479:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768095.411483:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294273792:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768095.411486:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768095.411488:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768095.411490:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768095.411492:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768095.411496:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f400. 02000000:00000001:2.0:1775768095.411498:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768095.411501:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768095.411505:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768095.411508:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768095.411511:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294273792, portal 4 00000100:00000001:2.0:1775768095.411514:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768095.411515:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641200128 : -112966068351488 : ffff9942099fce00) 00000100:00000040:2.0:1775768095.411520:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fce00 x1862022294273792/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768120 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768095.411528:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768095.411530:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768095.411533:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f880. 00000100:00000200:2.0:1775768095.411536:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294273792, offset 0 00000400:00000200:2.0:1775768095.411540:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768095.411544:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768095.411548:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768095.411551:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768095.411554:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768095.411557:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768095.411559:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768095.411563:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225312:225312:256:4294967295] 192.168.202.156@tcp LPNI seq info [225308:225308:8:4294967295] 00000400:00000200:2.0:1775768095.411567:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768095.411574:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768095.411579:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768095.411582:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3a00. 00000800:00000200:2.0:1775768095.411586:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768095.411590:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775768095.411594:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768095.411597:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1775768095.411601:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768095.411603:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3a00. 00000400:00000200:2.0:1775768095.411606:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768095.411610:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768095.411613:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f880 00000400:00000010:2.0:1775768095.411615:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f880. 00000100:00000001:2.0:1775768095.411617:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768095.411621:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fce00 x1862022294273792/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768120 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768095.411636:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768095.411639:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fce00 x1862022294273792/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768120 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768095.411647:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768095.411649:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768095.411652:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768095.411655:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:2.0:1775768095.411659:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768095.411661:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300020262 00000100:00000001:2.0:1775768095.411664:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300021565 00000100:00000001:2.0:1775768095.411667:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300020262 00000100:00000001:2.0:1775768095.411669:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300021565 00000100:00000001:2.0:1775768095.411672:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768095.411674:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768095.411676:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768095.411680:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768095/real 1775768095] req@ffff9942099fce00 x1862022294273792/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768095.411688:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768095.411689:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768095.411691:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768095.411693:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768095.411695:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768095.411697:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768095.411701:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fce00 x1862022294273792/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768095.411710:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fce00 x1862022294273792/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768095.411717:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768095.411719:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768095.411721:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768095.411723:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768095.411727:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fce00 x1862022294273792/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768095.411735:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768095.411739:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fce00 x1862022294273792/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768120 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768095.411747:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768095.411750:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768095.411752:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768095.411754:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768095.411756:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768095.411759:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768095.411761:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768095.411765:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768095.411768:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768095.411773:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fce00 x1862022294273792/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768120 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768095.411782:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294273792:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768095.411786:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768095.411788:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768095.411792:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fca80 x1862022294273408/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768120 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768095.411800:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768095.411802:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768095.411804:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f800. 02000000:00000001:2.0:1775768095.411807:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768095.411809:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768095.411811:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:2.0:1775768095.411813:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768095.411815:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028400. 02000000:00000001:2.0:1775768095.411818:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768095.411819:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768095.411821:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fca80. 00000100:00000001:2.0:1775768095.411824:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768095.411826:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768095.411828:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768095.411831:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fe680 x1862022294273536/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768120 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768095.411839:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768095.411840:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768095.411841:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1d000. 02000000:00000001:2.0:1775768095.411845:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768095.411846:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768095.411848:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:2.0:1775768095.411850:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768095.411851:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029800. 02000000:00000001:2.0:1775768095.411853:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768095.411854:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768095.411856:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fe680. 00000100:00000001:2.0:1775768095.411858:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768095.411860:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768095.411862:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768095.411866:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fdc00 x1862022294273664/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768120 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768095.411873:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768095.411874:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768095.411876:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1e400. 02000000:00000001:2.0:1775768095.411878:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768095.411879:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768095.411881:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:2.0:1775768095.411882:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768095.411884:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002a800. 02000000:00000001:2.0:1775768095.411886:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768095.411888:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768095.411889:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fdc00. 00000100:00000001:2.0:1775768095.411892:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768095.411894:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768095.411895:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768095.411899:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fce00 x1862022294273792/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768120 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768095.411906:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768095.411907:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768095.411908:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f400. 02000000:00000001:2.0:1775768095.411910:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768095.411911:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768095.411912:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:2.0:1775768095.411914:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768095.411916:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002bc00. 02000000:00000001:2.0:1775768095.411919:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768095.411920:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768095.411922:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fce00. 00000100:00000001:2.0:1775768095.411924:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768095.411926:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768095.411928:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768095.411931:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768095.411932:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768095.411934:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768095.411936:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768095.473246:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768095.473253:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768095.473258:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768095.473260:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768095.473263:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768095.473265:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768095.473268:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768095.473270:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768095.473274:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768095.473275:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768095.473340:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768095.473343:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768095.473347:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768095.537228:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768095.537236:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768095.537238:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768095.537244:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768095.537247:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768095.537252:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768095.537254:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768095.537257:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768095.537260:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768096.433278:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768096.433279:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768096.433283:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768096.433286:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768096.433287:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768096.433291:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768096.433291:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768096.433292:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768096.433296:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768096.433298:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:1.0:1775768096.433298:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:2.0:1775768096.433299:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768096.433304:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768096.433305:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768096.433305:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768096.433308:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768096.433310:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768096.433311:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768096.433311:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768096.433314:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768096.433318:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768096.433318:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768096.433321:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768096.433323:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768096.433326:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768096.433328:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768096.433331:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768096.497266:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768096.497271:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768096.497276:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768096.497278:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768096.497280:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768096.497283:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768096.497293:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768096.497296:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768096.497299:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768096.497301:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768096.497305:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768096.497307:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768096.497308:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768096.497310:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768096.497312:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768096.497313:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768096.497315:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768096.497317:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768096.497321:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768096.497322:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768096.497324:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768096.497326:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768096.497328:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768096.497329:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768096.497331:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768096.497333:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768096.497337:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768096.497338:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768096.497340:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768096.497341:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768096.497343:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768096.497344:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768096.497346:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768096.497348:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768096.497386:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768096.497390:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768096.497394:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768096.561247:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768096.561253:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768096.561255:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768096.561259:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768096.561262:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768096.561265:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768096.561267:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768096.561269:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768096.561271:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768096.753237:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4300021565, at 4300021568 00000100:00000040:0.0:1775768096.753239:0:14953:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4300021565, at 4300021568 00000100:00000010:1.0:1775768096.753251:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d6480. 00000100:00000010:0.0:1775768096.753253:0:14953:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d6300. 00000100:00000001:1.0:1775768097.457245:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768097.457246:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768097.457248:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768097.457251:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768097.457251:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768097.457253:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768097.457254:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768097.457254:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768097.457256:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768097.457262:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:1.0:1775768097.457262:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:2.0:1775768097.457263:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768097.457268:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768097.457269:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768097.457270:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768097.457272:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768097.457272:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768097.457273:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768097.457277:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768097.457280:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768097.457282:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768097.457284:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768097.457286:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768097.457289:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768097.457290:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768097.457293:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768097.457297:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768097.521255:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768097.521260:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768097.521266:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768097.521268:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768097.521272:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768097.521273:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768097.521277:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768097.521278:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768097.521281:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768097.521283:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768097.521334:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768097.521338:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768097.521340:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768097.585280:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768097.585287:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768097.585289:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768097.585295:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768097.585299:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768097.585303:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768097.585305:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768097.585308:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768097.585312:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768098.481266:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768098.481266:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768098.481266:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768098.481272:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768098.481273:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768098.481273:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768098.481274:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768098.481277:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768098.481277:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768098.481281:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:1.0:1775768098.481282:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:2.0:1775768098.481282:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768098.481288:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768098.481288:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768098.481295:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768098.481295:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768098.481296:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768098.481298:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768098.481301:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768098.481304:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768098.481308:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768098.481308:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768098.481312:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768098.481315:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768098.481316:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768098.481319:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768098.481322:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768098.545245:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768098.545252:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768098.545257:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768098.545259:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768098.545262:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768098.545263:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768098.545266:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768098.545267:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768098.545271:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768098.545272:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768098.545369:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768098.545373:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768098.545376:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768098.609250:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768098.609255:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768098.609256:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768098.609261:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768098.609264:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768098.609268:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768098.609270:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768098.609272:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768098.609274:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768099.505259:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768099.505267:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768099.505270:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768099.505275:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768099.505280:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768099.505286:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768099.505294:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768099.505297:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768099.505303:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768099.505455:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768099.505460:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768099.505463:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768099.505465:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:2.0:1775768099.505469:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768099.505470:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768099.505475:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768099.505477:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768099.505482:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768099.505483:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768099.505493:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768099.505494:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768099.505502:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768099.505503:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768099.505513:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768099.505613:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768099.505617:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768099.505621:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768099.569248:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768099.569253:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768099.569257:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768099.569259:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768099.569264:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768099.569265:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768099.569267:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768099.569269:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768099.569271:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768099.569272:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768099.569331:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768099.569335:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768099.569339:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768099.633272:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768099.633279:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768099.633281:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768099.633286:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768099.633290:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768099.633294:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768099.633297:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768099.633300:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768099.633303:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768100.529275:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768100.529275:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768100.529276:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768100.529282:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768100.529283:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768100.529284:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768100.529285:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768100.529287:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768100.529288:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768100.529290:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:0.0:1775768100.529295:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:1.0:1775768100.529295:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775768100.529295:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768100.529300:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768100.529302:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768100.529302:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768100.529303:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768100.529307:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768100.529308:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768100.529309:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768100.529310:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768100.529313:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768100.529316:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768100.529318:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768100.529322:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768100.529326:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768100.529330:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768100.529370:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768100.529375:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775768100.529377:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768100.529380:0:36:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768100.529383:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768100.529388:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5352 00000400:00000200:0.0:1775768100.529404:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768100.529409:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768100.529412:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768100.529414:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768100.529417:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768100.529418:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768100.529420:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768100.529423:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768100.529424:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:0.0:1775768100.529426:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768100.529428:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:0.0:1775768100.529430:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:0.0:1775768100.529432:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:0.0:1775768100.529433:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768100.529435:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768100.529436:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768100.529437:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768100.529440:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:0.0:1775768100.529442:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768100.529443:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768100.529446:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768100.529448:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768100.529451:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768100.529452:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775768100.529455:0:36:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768100.529463:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fea00. 00000020:00000040:0.0:1775768100.529467:0:36:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:0.0:1775768100.529471:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768100.529473:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768100.529476:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768100.529480:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768100.529485:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768100.529495:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220028000. 00000100:00000001:0.0:1775768100.529507:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768100.529517:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fea00 x1862022294273920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768100.529526:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768100.529529:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768100.529533:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008d4eaacc] to pc [ptlrpcd_rcv+-1] req@ffff9942099fea00 x1862022294273920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768100.529548:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768100.529550:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768100.529553:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775768100.529554:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768100.529555:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768100.529557:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768100.529559:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5352 00000400:00000200:0.0:1775768100.529562:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768100.529565:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768100.529567:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768100.529569:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768100.529571:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768100.529573:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768100.529575:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768100.529576:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768100.529577:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:0.0:1775768100.529579:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768100.529580:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:0.0:1775768100.529582:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:0.0:1775768100.529583:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000001:2.0:1775768100.529583:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768100.529588:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1775768100.529589:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768100.529592:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775768100.529593:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768100.529594:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768100.529597:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768100.529599:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00080000:0.0:1775768100.529603:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768100.529604:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768100.529604:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fea00 x1862022294273920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1775768100.529608:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768100.529613:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768100.529615:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1775768100.529616:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768100.529618:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768100.529619:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:2.0:1775768100.529624:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:0.0:1775768100.529625:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1775768100.529626:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1775768100.529628:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768100.529629:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768100.529631:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768100.529635:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294273920:192.168.202.156@tcp:250:kworker.0 00000100:00000010:0.0:1775768100.529636:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fce00. 00000020:00000040:0.0:1775768100.529641:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000100:00000001:2.0:1775768100.529641:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768100.529645:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775768100.529646:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768100.529647:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768100.529649:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768100.529650:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768100.529653:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775768100.529654:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1775768100.529655:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775768100.529659:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f400. 02000000:00000010:0.0:1775768100.529661:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220029400. 02000000:00000001:2.0:1775768100.529663:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768100.529665:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768100.529671:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00100000:0.0:1775768100.529672:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fce00 x1862022294274048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:2.0:1775768100.529676:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000100:00000001:0.0:1775768100.529682:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000010:2.0:1775768100.529683:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000001:0.0:1775768100.529684:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768100.529689:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000060e5e9d1] to pc [ptlrpcd_rcv+-1] req@ffff9942099fce00 x1862022294274048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768100.529691:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294273920, portal 25 00000100:00000001:2.0:1775768100.529696:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768100.529699:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641207296 : -112966068344320 : ffff9942099fea00) 00000100:00000001:0.0:1775768100.529700:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768100.529705:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768100.529708:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775768100.529711:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768100.529712:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:2.0:1775768100.529712:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fea00 x1862022294273920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768125 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768100.529717:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768100.529721:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5352 00000400:00000200:0.0:1775768100.529729:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768100.529735:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768100.529736:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1775768100.529736:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768100.529741:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768100.529745:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768100.529745:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000100:00000040:0.0:1775768100.529749:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768100.529751:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000010:2.0:1775768100.529752:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f880. 00000020:00000001:0.0:1775768100.529754:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768100.529757:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000100:00000200:2.0:1775768100.529758:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294273920, offset 0 00000020:00000001:0.0:1775768100.529759:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768100.529763:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:0.0:1775768100.529766:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000400:00000200:2.0:1775768100.529769:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000020:00000001:0.0:1775768100.529772:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:0.0:1775768100.529777:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768100.529779:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768100.529782:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768100.529785:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000200:2.0:1775768100.529788:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000040:0.0:1775768100.529793:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000400:00000200:2.0:1775768100.529798:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00080000:0.0:1775768100.529799:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768100.529804:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768100.529804:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:0.0:1775768100.529809:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:2.0:1775768100.529811:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 02000000:00000001:0.0:1775768100.529815:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1775768100.529815:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 02000000:00000001:0.0:1775768100.529821:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768100.529821:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00000001:0.0:1775768100.529824:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775768100.529827:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775768100.529832:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768100.529832:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225313:225313:256:4294967295] 192.168.202.156@tcp LPNI seq info [225309:225309:8:4294967295] 00000100:00000001:0.0:1775768100.529835:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768100.529839:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fdc00. 00000020:00000040:0.0:1775768100.529842:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:0.0:1775768100.529847:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768100.529850:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768100.529851:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768100.529852:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768100.529854:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768100.529857:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768100.529859:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002bc00. 00000100:00000001:0.0:1775768100.529862:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768100.529870:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fdc00 x1862022294274176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1775768100.529872:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768100.529888:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00000001:0.0:1775768100.529889:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768100.529894:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1775768100.529896:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3a00. 00000100:00000040:0.0:1775768100.529900:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001372d9ae] to pc [ptlrpcd_rcv+-1] req@ffff9942099fdc00 x1862022294274176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1775768100.529909:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:0.0:1775768100.529918:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775768100.529919:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00080000:0.0:1775768100.529938:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768100.529943:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775768100.529945:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768100.529947:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768100.529949:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775768100.529949:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768100.529955:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5352 00000400:00000200:2.0:1775768100.529956:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768100.529961:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4300026685 00000400:00000200:0.0:1775768100.529964:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000010:2.0:1775768100.529967:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d6f00. 00000400:00000200:0.0:1775768100.529968:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768100.529972:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768100.529975:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768100.529976:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775768100.529977:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000040:0.0:1775768100.529980:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768100.529983:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:2.0:1775768100.529984:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4300026685 00000020:00000001:0.0:1775768100.529986:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768100.529988:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000100:00000010:2.0:1775768100.529989:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d6720. 00000020:00000001:0.0:1775768100.529990:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768100.529993:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000100:00000001:2.0:1775768100.529995:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000020:00000001:0.0:1775768100.529996:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:0.0:1775768100.529999:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:2.0:1775768100.530000:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1775768100.530004:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768100.530004:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768100.530009:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775768100.530010:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768100.530013:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768100.530014:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768100.530018:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:2.0:1775768100.530019:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000020:00000040:0.0:1775768100.530021:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00000001:2.0:1775768100.530021:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1775768100.530026:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768100.530029:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1775768100.530030:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775768100.530033:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1775768100.530034:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768100.530038:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:2.0:1775768100.530041:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fce00 x1862022294274048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1775768100.530043:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768100.530050:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768100.530055:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775768100.530059:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000100:00000001:2.0:1775768100.530060:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000008:00000001:0.0:1775768100.530064:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768100.530064:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768100.530070:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768100.530071:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000010:0.0:1775768100.530075:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fe680. 02000000:00000001:2.0:1775768100.530075:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:0.0:1775768100.530081:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:2.0:1775768100.530081:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768100.530087:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768100.530090:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768100.530090:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294274048:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768100.530094:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768100.530097:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1775768100.530098:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768100.530101:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768100.530101:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768100.530104:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768100.530106:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002a800. 02000000:00000001:2.0:1775768100.530106:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1775768100.530112:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775768100.530112:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1e400. 02000000:00000001:2.0:1775768100.530116:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768100.530121:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fe680 x1862022294274304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1775768100.530121:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768100.530123:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39790. 00000400:00000010:2.0:1775768100.530128:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8fe58. 00000100:00000200:2.0:1775768100.530133:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294274048, portal 10 00000100:00000001:2.0:1775768100.530138:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768100.530140:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775768100.530140:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641200128 : -112966068351488 : ffff9942099fce00) 00000100:00000001:0.0:1775768100.530145:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768100.530149:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fce00 x1862022294274048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768125 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775768100.530153:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ba41ba08] to pc [ptlrpcd_rcv+-1] req@ffff9942099fe680 x1862022294274304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768100.530181:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768100.530185:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768100.530188:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8faa0. 00000100:00000200:2.0:1775768100.530194:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294274048, offset 0 00000400:00000200:2.0:1775768100.530204:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00000001:0.0:1775768100.530205:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768100.530210:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:2.0:1775768100.530212:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00080000:0.0:1775768100.530217:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775768100.530221:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:2.0:1775768100.530222:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00080000:0.0:1775768100.530225:0:36:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:2.0:1775768100.530228:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1775768100.530230:0:36:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:2.0:1775768100.530232:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000100:00080000:0.0:1775768100.530236:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5352 00000400:00000200:2.0:1775768100.530240:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768100.530245:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768100.530247:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768100.530252:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225314:225314:255:4294967295] 192.168.202.156@tcp LPNI seq info [225310:225310:7:4294967295] 00000400:00000200:0.0:1775768100.530256:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768100.530261:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1775768100.530263:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000040:0.0:1775768100.530268:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768100.530272:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768100.530278:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000200:2.0:1775768100.530283:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000100:00000001:0.0:1775768100.530284:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768100.530290:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768100.530292:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000800:00000200:2.0:1775768100.530296:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000020:00000001:0.0:1775768100.530297:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768100.530300:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000800:00000010:2.0:1775768100.530303:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3500. 00000020:00000001:0.0:1775768100.530304:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:0.0:1775768100.530309:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000800:00000200:2.0:1775768100.530314:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000040:0.0:1775768100.530317:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768100.530321:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768100.530326:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000800:00000200:2.1:1775768100.530327:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:0.0:1775768100.530329:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:2.0:1775768100.530332:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1775768100.530334:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00000040:2.0:1775768100.530339:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000100:00080000:0.0:1775768100.530340:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768100.530346:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768100.530349:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:0.0:1775768100.530351:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1775768100.530354:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300025363 02000000:00000001:0.0:1775768100.530356:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768100.530361:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768100.530364:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775768100.530366:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:0.0:1775768100.530368:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768100.530370:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300026685 00000100:00000001:0.0:1775768100.530371:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768100.530377:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300025363 00000100:00000010:0.0:1775768100.530379:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fca80. 00000020:00000040:0.0:1775768100.530384:0:36:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00000100:00000001:2.0:1775768100.530385:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300026685 02000000:00000001:0.0:1775768100.530386:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768100.530390:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768100.530391:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768100.530394:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1775768100.530396:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768100.530398:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1775768100.530400:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1775768100.530402:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768100.530407:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fdc00 x1862022294274176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1775768100.530409:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220029800. 00000100:00000001:0.0:1775768100.530416:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768100.530422:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768100.530426:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768100.530427:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fca80 x1862022294274432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1775768100.530431:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768100.530434:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768100.530439:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768100.530447:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294274176:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768100.530448:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775768100.530453:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1775768100.530454:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768100.530457:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768100.530460:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768100.530463:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000046a3ca2f] to pc [ptlrpcd_rcv+-1] req@ffff9942099fca80 x1862022294274432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1775768100.530463:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768100.530469:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1d000. 02000000:00000001:2.0:1775768100.530474:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768100.530484:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768100.530485:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775768100.530488:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39898. 00000100:00000040:0.0:1775768100.530492:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5363) 00000400:00000010:2.0:1775768100.530494:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768100.530499:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294274176, portal 10 00000100:00000001:2.0:1775768100.530501:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768100.530502:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641203712 : -112966068347904 : ffff9942099fdc00) 00000100:00000040:2.0:1775768100.530511:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fdc00 x1862022294274176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768125 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768100.530518:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768100.530520:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768100.530523:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f660. 00000100:00000200:2.0:1775768100.530526:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294274176, offset 0 00000400:00000200:2.0:1775768100.530530:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768100.530534:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768100.530538:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768100.530541:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768100.530543:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768100.530545:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768100.530547:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768100.530550:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225315:225315:254:4294967295] 192.168.202.156@tcp LPNI seq info [225311:225311:6:4294967295] 00000400:00000200:2.0:1775768100.530554:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768100.530560:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768100.530564:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768100.530568:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3800. 00000800:00000200:2.0:1775768100.530571:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768100.530575:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:2.0:1775768100.530577:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768100.530580:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:2.0:1775768100.530583:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768100.530585:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300025355 00000100:00000001:2.0:1775768100.530587:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300026685 00000100:00000001:2.0:1775768100.530590:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300025355 00000100:00000001:2.0:1775768100.530592:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300026685 00000100:00000001:2.0:1775768100.530594:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768100.530595:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768100.530598:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768100.530600:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768100.530604:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768100.530606:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1775768100.530608:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768100.530609:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768100.530610:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768100.530614:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fe680 x1862022294274304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768100.530622:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768100.530624:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768100.530626:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768100.530628:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768100.530630:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768100.530633:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294274304:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768100.530637:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768100.530638:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768100.530640:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768100.530642:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768100.530645:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f800. 02000000:00000001:2.0:1775768100.530647:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768100.530651:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768100.530655:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39f20. 00000400:00000010:2.0:1775768100.530659:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f3b8. 00000100:00000200:2.0:1775768100.530662:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294274304, portal 4 00000100:00000001:2.0:1775768100.530664:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768100.530665:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641206400 : -112966068345216 : ffff9942099fe680) 00000100:00000040:2.0:1775768100.530671:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fe680 x1862022294274304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768125 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768100.530678:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768100.530680:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768100.530682:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f550. 00000100:00000200:2.0:1775768100.530685:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294274304, offset 0 00000400:00000200:2.0:1775768100.530689:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768100.530694:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768100.530697:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768100.530700:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768100.530703:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768100.530705:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768100.530708:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768100.530711:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225316:225316:253:4294967295] 192.168.202.156@tcp LPNI seq info [225312:225312:5:4294967295] 00000400:00000200:2.0:1775768100.530716:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768100.530723:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768100.530727:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768100.530731:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3600. 00000800:00000200:2.0:1775768100.530735:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768100.530739:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:2.0:1775768100.530742:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768100.530745:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:2.0:1775768100.530749:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768100.530752:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300025378 00000100:00000001:2.0:1775768100.530754:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300026685 00000100:00000001:2.0:1775768100.530757:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300025378 00000100:00000001:2.0:1775768100.530759:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300026685 00000100:00000001:2.0:1775768100.530761:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768100.530764:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768100.530766:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768100.530771:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fca80 x1862022294274432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768100.530778:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768100.530779:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768100.530782:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768100.530783:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768100.530785:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768100.530787:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294274432:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768100.530790:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768100.530792:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768100.530794:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768100.530795:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768100.530799:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f000. 02000000:00000001:2.0:1775768100.530801:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768100.530805:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768100.530807:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39370. 00000400:00000010:2.0:1775768100.530811:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f4c8. 00000100:00000200:2.0:1775768100.530814:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294274432, portal 4 00000100:00000001:2.0:1775768100.530816:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768100.530817:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641199232 : -112966068352384 : ffff9942099fca80) 00000100:00000040:2.0:1775768100.530822:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fca80 x1862022294274432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768125 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768100.530829:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768100.530832:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768100.530835:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8fc38. 00000100:00000200:2.0:1775768100.530837:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294274432, offset 0 00000400:00000200:2.0:1775768100.530841:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768100.530846:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768100.530849:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768100.530852:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768100.530855:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768100.530857:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768100.530860:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768100.530863:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225317:225317:252:4294967295] 192.168.202.156@tcp LPNI seq info [225313:225313:4:4294967295] 00000400:00000200:2.0:1775768100.530867:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768100.530873:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768100.530877:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768100.530881:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3300. 00000800:00000200:2.0:1775768100.530884:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768100.530888:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:2.0:1775768100.530891:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768100.530893:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:2.0:1775768100.530896:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768100.530898:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300025381 00000100:00000001:2.0:1775768100.530900:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300026685 00000100:00000001:2.0:1775768100.530903:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300025381 00000100:00000001:2.0:1775768100.530905:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300026685 00000100:00000001:2.0:1775768100.530907:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768100.530909:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768100.530911:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768100.530913:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768100.530917:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768100.530918:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1775768100.530920:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768100.530936:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768100.530938:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768100.530940:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768100.593272:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768100.593277:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768100.593281:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768100.593282:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768100.593285:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768100.593286:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768100.593288:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768100.593289:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768100.593292:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768100.593293:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768100.593339:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768100.593342:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768100.593344:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768100.657297:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768100.657304:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768100.657306:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768100.657312:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768100.657316:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768100.657321:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768100.657324:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768100.657326:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768100.657329:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768101.553225:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768101.553228:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768101.553229:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768101.553232:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768101.553233:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768101.553235:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768101.553237:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:1.0:1775768101.553237:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:0.0:1775768101.553242:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768101.553243:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768101.553247:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768101.553247:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768101.553250:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768101.553252:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768101.553254:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768101.553258:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768101.553260:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768101.553262:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768101.553317:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768101.553319:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768101.553320:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768101.553322:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768101.553323:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768101.553324:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768101.553326:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768101.553327:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768101.553328:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768101.617262:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768101.617268:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768101.617271:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768101.617274:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768101.617278:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768101.617281:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768101.617283:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768101.617284:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768101.617288:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768101.617290:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768101.617293:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768101.617294:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768101.617296:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768101.617298:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768101.617301:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768101.617302:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768101.617346:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768101.617349:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768101.617352:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768101.681262:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768101.681268:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768101.681270:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768101.681274:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768101.681278:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768101.681282:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768101.681284:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768101.681287:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768101.681290:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768101.873239:0:14953:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4300026685, at 4300026688 00000100:00000040:1.0:1775768101.873239:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4300026685, at 4300026688 00000100:00000010:1.0:1775768101.873252:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d6720. 00000100:00000010:0.0:1775768101.873254:0:14953:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d6f00. 00000100:00000001:0.0:1775768102.577250:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768102.577250:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768102.577250:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768102.577258:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768102.577258:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768102.577258:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768102.577261:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768102.577262:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768102.577262:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768102.577269:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:1.0:1775768102.577269:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:2.0:1775768102.577269:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768102.577275:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768102.577275:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768102.577276:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768102.577281:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768102.577281:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768102.577281:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768102.577286:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768102.577289:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768102.577291:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768102.577293:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768102.577296:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768102.577299:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768102.577299:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768102.577303:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768102.577306:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768102.641255:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768102.641259:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768102.641263:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768102.641264:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768102.641266:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768102.641267:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768102.641269:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768102.641270:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768102.641272:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768102.641273:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768102.641324:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768102.641328:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768102.641331:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768102.705236:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768102.705241:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768102.705243:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768102.705247:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768102.705249:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768102.705252:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768102.705254:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768102.705256:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768102.705257:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768103.601227:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768103.601227:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768103.601233:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768103.601234:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768103.601235:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768103.601237:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768103.601241:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:2.0:1775768103.601241:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768103.601246:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768103.601246:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768103.601249:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768103.601250:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768103.601252:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768103.601254:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768103.601257:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768103.601260:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768103.601262:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768103.601265:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768103.601390:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768103.601394:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768103.601395:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768103.601397:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768103.601398:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768103.601400:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768103.601401:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000400:00000100:1.0:1775768103.601402:0:2388:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.156:988 00000100:00000001:3.0:1775768103.601403:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768103.601406:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:1.0:1775768103.601440:0:2388:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.156@tcp at host 192.168.202.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:1.0:1775768103.601450:0:2388:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768103.601454:0:2388:0:(router.c:1750:lnet_notify()) 192.168.202.56@tcp notifying 192.168.202.156@tcp: down 00000400:00000200:1.0:1775768103.601458:0:2388:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768103.601462:0:2388:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:1.0:1775768103.601467:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000800:00000010:1.0:1775768103.601471:0:2388:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3a00. 00000400:00000200:1.0:1775768103.601476:0:2388:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775768103.601480:0:2388:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775768103.601486:0:2388:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f880 00000400:00000010:1.0:1775768103.601488:0:2388:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f880. 00000100:00000001:1.0:1775768103.601493:0:2388:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768103.601501:0:2388:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9942099fea00 x1862022294273920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768125 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768103.601527:0:2388:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768103.601532:0:2388:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fea00 x1862022294273920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768125 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768103.601540:0:2388:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768103.601541:0:2388:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775768103.601551:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000800:00000010:1.0:1775768103.601554:0:2388:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3500. 00000400:00000200:1.0:1775768103.601556:0:2388:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775768103.601559:0:2388:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775768103.601563:0:2388:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8faa0 00000400:00000010:1.0:1775768103.601565:0:2388:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8faa0. 00000100:00000001:1.0:1775768103.601567:0:2388:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768103.601571:0:2388:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9942099fce00 x1862022294274048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768125 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768103.601579:0:2388:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768103.601581:0:2388:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fce00 x1862022294274048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768125 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768103.601587:0:2388:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768103.601589:0:2388:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775768103.601593:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000800:00000010:1.0:1775768103.601595:0:2388:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3800. 00000400:00000200:1.0:1775768103.601598:0:2388:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775768103.601601:0:2388:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775768103.601603:0:2388:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f660 00000400:00000010:1.0:1775768103.601604:0:2388:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f660. 00000100:00000001:1.0:1775768103.601606:0:2388:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768103.601610:0:2388:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9942099fdc00 x1862022294274176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768125 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768103.601616:0:2388:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768103.601619:0:2388:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fdc00 x1862022294274176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768125 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768103.601624:0:2388:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768103.601626:0:2388:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775768103.601630:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000800:00000010:1.0:1775768103.601633:0:2388:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3600. 00000400:00000200:1.0:1775768103.601635:0:2388:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775768103.601638:0:2388:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775768103.601640:0:2388:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f550 00000400:00000010:1.0:1775768103.601641:0:2388:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f550. 00000100:00000001:1.0:1775768103.601643:0:2388:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768103.601647:0:2388:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9942099fe680 x1862022294274304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768125 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768103.601654:0:2388:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768103.601656:0:2388:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fe680 x1862022294274304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768125 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768103.601661:0:2388:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768103.601662:0:2388:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775768103.601665:0:2388:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000800:00000010:1.0:1775768103.601667:0:2388:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3300. 00000400:00000200:1.0:1775768103.601669:0:2388:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775768103.601672:0:2388:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775768103.601675:0:2388:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8fc38 00000400:00000010:1.0:1775768103.601676:0:2388:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8fc38. 00000100:00000001:1.0:1775768103.601677:0:2388:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768103.601682:0:2388:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9942099fca80 x1862022294274432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768125 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768103.601688:0:2388:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768103.601690:0:2388:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fca80 x1862022294274432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768125 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768103.601695:0:2388:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768103.601696:0:2388:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768103.601750:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768103.601754:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768103.601757:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768103.601764:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768100/real 1775768103] req@ffff9942099fe680 x1862022294274304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768103.601775:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768103.601777:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768103.601780:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39f20. 00000400:00000200:2.0:1775768103.601784:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f3b8 00000400:00000010:2.0:1775768103.601785:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f3b8. 00000100:00000001:2.0:1775768103.601788:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768103.601792:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fe680 x1862022294274304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768103.601800:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fe680 x1862022294274304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768103.601807:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768103.601808:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768103.601827:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768103.601829:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768103.601833:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fe680 x1862022294274304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768103.601841:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768103.601845:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fe680 x1862022294274304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768125 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768103.601852:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768103.601855:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768103.601857:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768103.601858:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768103.601861:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768103.601866:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768103.601871:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768103.601876:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768103.601878:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768103.601883:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fe680 x1862022294274304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768125 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768103.601905:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294274304:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768103.601911:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768103.601913:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768100/real 1775768103] req@ffff9942099fca80 x1862022294274432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768103.601919:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768103.601921:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768103.601922:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39370. 00000400:00000200:2.0:1775768103.601925:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f4c8 00000400:00000010:2.0:1775768103.601926:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f4c8. 00000100:00000001:2.0:1775768103.601928:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768103.601931:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fca80 x1862022294274432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768103.601937:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fca80 x1862022294274432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768103.601944:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768103.601946:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768103.601947:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768103.601948:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768103.601952:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fca80 x1862022294274432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768103.601958:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768103.601961:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fca80 x1862022294274432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768125 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768103.601973:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768103.601975:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768103.601977:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768103.601978:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768103.601980:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768103.601983:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768103.601986:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768103.601989:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768103.601991:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768103.602001:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fca80 x1862022294274432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768125 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768103.602009:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294274432:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768103.602014:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768103.602018:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768100/real 1775768103] req@ffff9942099fce00 x1862022294274048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768103.602025:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768103.602026:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768103.602028:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39790. 00000400:00000200:2.0:1775768103.602030:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8fe58 00000400:00000010:2.0:1775768103.602032:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8fe58. 00000100:00000001:2.0:1775768103.602034:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768103.602036:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fce00 x1862022294274048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768103.602043:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fce00 x1862022294274048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768103.602048:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768103.602050:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768103.602052:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768103.602054:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768103.602057:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fce00 x1862022294274048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768103.602063:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768103.602066:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fce00 x1862022294274048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768125 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768103.602073:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768103.602074:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768103.602076:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768103.602077:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768103.602079:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768103.602082:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768103.602084:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768103.602087:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768103.602089:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768103.602092:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fce00 x1862022294274048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768125 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768103.602099:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294274048:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768103.602102:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768103.602105:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768100/real 1775768103] req@ffff9942099fdc00 x1862022294274176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768103.602113:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768103.602115:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768103.602116:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39898. 00000400:00000200:2.0:1775768103.602118:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768103.602120:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768103.602122:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768103.602129:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fdc00 x1862022294274176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768103.602136:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fdc00 x1862022294274176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768103.602141:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768103.602143:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768103.602144:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768103.602145:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768103.602148:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fdc00 x1862022294274176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768103.602912:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768103.602925:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fdc00 x1862022294274176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768125 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768103.602933:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768103.602935:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768103.602937:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768103.602938:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768103.602940:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768103.602943:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768103.602945:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768103.602948:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768103.602950:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768103.602953:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fdc00 x1862022294274176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768125 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768103.602960:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294274176:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768103.602964:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768103.602968:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768100/real 1775768103] req@ffff9942099fea00 x1862022294273920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768103.602976:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768103.602977:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768103.602979:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768103.602982:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768103.602983:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768103.602986:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768103.602989:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fea00 x1862022294273920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768103.602996:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fea00 x1862022294273920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768103.603002:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768103.603003:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768103.603005:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768103.603006:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768103.603009:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fea00 x1862022294273920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768103.603014:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768103.603017:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fea00 x1862022294273920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768125 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768103.603023:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768103.603024:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768103.603026:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768103.603027:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768103.603028:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768103.603030:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768103.603032:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768103.603035:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768103.603037:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768103.603040:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fea00 x1862022294273920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768125 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768103.603047:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294273920:192.168.202.156@tcp:250:kworker.0 00000100:00000001:2.0:1775768103.603051:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768103.603053:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768103.603056:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fe680 x1862022294274304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768125 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768103.603062:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768103.603064:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768103.603068:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f800. 02000000:00000001:2.0:1775768103.603071:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768103.603073:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768103.603075:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:2.0:1775768103.603077:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768103.603080:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002a800. 02000000:00000001:2.0:1775768103.603084:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768103.603086:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768103.603087:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fe680. 00000100:00000001:2.0:1775768103.603091:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768103.603092:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768103.603093:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768103.603096:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fca80 x1862022294274432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768125 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768103.603101:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768103.603102:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768103.603103:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f000. 02000000:00000001:2.0:1775768103.603105:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768103.603106:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768103.603107:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:2.0:1775768103.603109:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768103.603110:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029800. 02000000:00000001:2.0:1775768103.603112:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768103.603113:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768103.603114:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fca80. 00000100:00000001:2.0:1775768103.603116:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768103.603117:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768103.603118:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768103.603121:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fce00 x1862022294274048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768125 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768103.603126:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768103.603127:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768103.603129:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1e400. 02000000:00000001:2.0:1775768103.603131:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768103.603132:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768103.603133:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:2.0:1775768103.603135:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768103.603136:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029400. 02000000:00000001:2.0:1775768103.603138:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768103.603139:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768103.603140:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fce00. 00000100:00000001:2.0:1775768103.603142:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768103.603144:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768103.603145:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768103.603148:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fdc00 x1862022294274176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768125 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768103.603179:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768103.603181:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768103.603183:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1d000. 02000000:00000001:2.0:1775768103.603186:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768103.603187:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768103.603188:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:2.0:1775768103.603190:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768103.603191:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002bc00. 02000000:00000001:2.0:1775768103.603194:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768103.603195:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768103.603196:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fdc00. 00000100:00000001:2.0:1775768103.603198:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768103.603199:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768103.603201:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768103.603204:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fea00 x1862022294273920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768125 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768103.603210:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768103.603211:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768103.603213:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f400. 02000000:00000001:2.0:1775768103.603215:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768103.603216:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768103.603218:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:2.0:1775768103.603220:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768103.603221:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028000. 02000000:00000001:2.0:1775768103.603224:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768103.603225:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768103.603226:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fea00. 00000100:00000001:2.0:1775768103.603228:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768103.603229:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768103.603232:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768103.603242:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768103.603245:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768103.603248:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768103.603250:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768103.665223:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768103.665228:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768103.665232:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768103.665233:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768103.665235:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768103.665236:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768103.665238:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768103.665238:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768103.665240:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768103.665241:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768103.665278:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768103.665281:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768103.665283:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768103.729255:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768103.729259:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768103.729260:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768103.729264:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768103.729267:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768103.729270:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768103.729272:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768103.729273:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768103.729275:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768104.625219:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768104.625219:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768104.625226:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768104.625226:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768104.625229:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768104.625229:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768104.625235:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:2.0:1775768104.625235:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768104.625240:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768104.625240:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768104.625244:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768104.625244:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768104.625249:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768104.625251:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768104.625253:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768104.625258:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768104.625260:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768104.625262:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768104.625272:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768104.625276:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768104.625277:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768104.625280:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768104.625283:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768104.625284:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768104.625286:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768104.625287:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768104.625288:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768104.689251:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768104.689255:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768104.689259:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768104.689260:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768104.689262:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768104.689263:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768104.689265:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768104.689266:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768104.689268:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768104.689269:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768104.689311:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768104.689314:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768104.689316:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768104.753225:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768104.753232:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768104.753234:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768104.753240:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768104.753244:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768104.753248:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768104.753251:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768104.753253:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768104.753256:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768105.649271:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768105.649271:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768105.649277:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768105.649278:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768105.649280:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768105.649282:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775768105.649282:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768105.649288:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:2.0:1775768105.649288:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00080000:0.0:1775768105.649289:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775768105.649295:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775768105.649295:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768105.649296:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768105.649299:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:0.0:1775768105.649300:0:36:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768105.649301:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768105.649302:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768105.649304:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768105.649307:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775768105.649308:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768105.649309:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768105.649312:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775768105.649314:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00080000:0.0:1775768105.649317:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5358 00000100:00000001:1.0:1775768105.649319:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768105.649319:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768105.649320:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768105.649326:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1775768105.649332:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768105.649332:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768105.649338:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768105.649338:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000400:00000200:0.0:1775768105.649343:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:3.0:1775768105.649344:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1775768105.649349:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1775768105.649349:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768105.649357:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768105.649362:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768105.649366:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768105.649369:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768105.649373:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768105.649376:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:0.0:1775768105.649379:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768105.649382:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:0.0:1775768105.649386:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:0.0:1775768105.649389:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:0.0:1775768105.649393:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768105.649395:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768105.649398:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768105.649400:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768105.649404:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:0.0:1775768105.649407:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768105.649410:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768105.649413:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768105.649417:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768105.649420:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768105.649422:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775768105.649425:0:36:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768105.649431:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fc380. 00000020:00000040:0.0:1775768105.649435:0:36:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:0.0:1775768105.649439:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768105.649441:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768105.649443:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768105.649446:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768105.649451:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768105.649461:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002b800. 00000100:00000001:0.0:1775768105.649468:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768105.649478:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fc380 x1862022294274560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768105.649490:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768105.649494:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768105.649516:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002957419c] to pc [ptlrpcd_rcv+-1] req@ffff9942099fc380 x1862022294274560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768105.649549:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768105.649553:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768105.649556:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775768105.649558:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768105.649559:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768105.649562:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768105.649565:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5358 00000400:00000200:0.0:1775768105.649569:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768105.649573:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768105.649576:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768105.649579:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768105.649581:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768105.649583:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768105.649585:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768105.649588:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768105.649589:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:0.0:1775768105.649591:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768105.649593:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:0.0:1775768105.649595:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:0.0:1775768105.649597:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:0.0:1775768105.649600:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768105.649603:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768105.649605:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768105.649607:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768105.649609:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00080000:0.0:1775768105.649612:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768105.649615:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768105.649617:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768105.649629:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768105.649631:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768105.649633:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775768105.649638:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775768105.649642:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768105.649644:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768105.649648:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fed80. 00000020:00000040:0.0:1775768105.649651:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:0.0:1775768105.649654:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768105.649655:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768105.649657:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768105.649659:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768105.649661:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768105.649665:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220028000. 00000100:00000001:0.0:1775768105.649670:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768105.649675:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fed80 x1862022294274688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768105.649684:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768105.649686:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768105.649690:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000c09b4d8] to pc [ptlrpcd_rcv+-1] req@ffff9942099fed80 x1862022294274688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768105.649699:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768105.649701:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768105.649705:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775768105.649706:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768105.649708:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768105.649710:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768105.649713:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5358 00000400:00000200:0.0:1775768105.649717:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768105.649720:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768105.649722:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768105.649725:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768105.649728:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768105.649730:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768105.649732:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768105.649734:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768105.649735:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:0.0:1775768105.649737:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768105.649739:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:0.0:1775768105.649741:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:0.0:1775768105.649744:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:0.0:1775768105.649748:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768105.649750:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768105.649752:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768105.649754:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768105.649757:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:0.0:1775768105.649760:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768105.649762:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768105.649764:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768105.649767:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768105.649770:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768105.649772:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775768105.649774:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775768105.649777:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768105.649779:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768105.649783:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fd180. 00000020:00000040:0.0:1775768105.649786:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:0.0:1775768105.649788:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768105.649790:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768105.649792:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768105.649793:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768105.649795:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768105.649799:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002bc00. 00000100:00000001:0.0:1775768105.649802:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768105.649808:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fd180 x1862022294274816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768105.649816:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768105.649818:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768105.649823:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004ec16d17] to pc [ptlrpcd_rcv+-1] req@ffff9942099fd180 x1862022294274816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768105.649832:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768105.649835:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768105.649839:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775768105.649841:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768105.649842:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768105.649845:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768105.649847:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5358 00000400:00000200:0.0:1775768105.649852:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768105.649855:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768105.649857:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768105.649861:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768105.649863:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768105.649865:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768105.649867:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768105.649870:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768105.649902:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:0.0:1775768105.649905:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768105.649907:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:0.0:1775768105.649909:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:0.0:1775768105.649912:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:0.0:1775768105.649915:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768105.649917:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768105.649919:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768105.649922:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768105.649925:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:0.0:1775768105.649928:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768105.649930:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768105.649932:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768105.649935:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768105.649937:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768105.649939:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775768105.649942:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:0.0:1775768105.649945:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768105.649946:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768105.649950:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ff480. 00000020:00000040:0.0:1775768105.649953:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:0.0:1775768105.649956:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768105.649957:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768105.649959:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768105.649960:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768105.649962:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768105.649966:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220029400. 00000100:00000001:0.0:1775768105.649970:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768105.649975:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ff480 x1862022294274944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768105.649984:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768105.649986:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768105.649990:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e73dcd12] to pc [ptlrpcd_rcv+-1] req@ffff9942099ff480 x1862022294274944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768105.649999:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768105.650002:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768105.650005:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775768105.650006:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768105.650008:0:36:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768105.650011:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768105.650014:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5358 00000400:00000200:0.0:1775768105.650019:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768105.650022:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768105.650025:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768105.650028:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768105.650030:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768105.650032:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768105.650035:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768105.650037:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768105.650039:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:0.0:1775768105.650040:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768105.650042:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:0.0:1775768105.650044:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:0.0:1775768105.650047:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:0.0:1775768105.650049:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768105.650051:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768105.650054:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768105.650056:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768105.650059:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:0.0:1775768105.650061:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768105.650064:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768105.650066:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768105.650070:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768105.650072:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768105.650074:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775768105.650076:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:0.0:1775768105.650079:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768105.650081:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768105.650086:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fd880. 00000020:00000040:0.0:1775768105.650088:0:36:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:0.0:1775768105.650091:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768105.650099:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768105.650101:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768105.650103:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768105.650105:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768105.650109:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220029800. 00000100:00000001:0.0:1775768105.650113:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768105.650117:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fd880 x1862022294275072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768105.650132:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768105.650134:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768105.650139:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007f8a5f3b] to pc [ptlrpcd_rcv+-1] req@ffff9942099fd880 x1862022294275072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768105.650147:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768105.650152:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5368) 00000100:00000001:2.0:1775768105.650250:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768105.650254:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768105.650256:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768105.650262:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fc380 x1862022294274560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768105.650271:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768105.650273:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768105.650276:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768105.650279:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768105.650282:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768105.650286:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294274560:192.168.202.156@tcp:250:kworker.0 00000100:00000001:2.0:1775768105.650290:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768105.650294:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768105.650297:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768105.650300:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768105.650305:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f400. 02000000:00000001:2.0:1775768105.650308:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768105.650314:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768105.650319:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768105.650326:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768105.650333:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294274560, portal 25 00000100:00000001:2.0:1775768105.650336:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768105.650338:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641197440 : -112966068354176 : ffff9942099fc380) 00000100:00000040:2.0:1775768105.650344:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fc380 x1862022294274560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768130 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768105.650352:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768105.650355:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768105.650359:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768105.650363:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294274560, offset 0 00000400:00000200:2.0:1775768105.650371:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768105.650382:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768105.650389:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768105.650396:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768105.650401:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768105.650405:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768105.650408:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768105.650413:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225318:225318:256:4294967295] 192.168.202.156@tcp LPNI seq info [225314:225314:8:4294967295] 00000400:00000200:2.0:1775768105.650419:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768105.650432:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768105.650439:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768105.650445:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768105.650453:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768105.650456:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775768105.650463:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768105.650467:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1775768105.650472:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768105.650475:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768105.650481:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768105.650486:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768105.650492:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768105.650494:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768105.650499:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768105.650503:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fc380 x1862022294274560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768130 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768105.650513:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768105.650516:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fc380 x1862022294274560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768130 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768105.650524:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768105.650526:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768105.650532:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768105.650536:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768105.650539:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4300031806 00000100:00000010:2.0:1775768105.650544:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d6a80. 00000100:00000001:2.0:1775768105.650552:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1775768105.650558:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4300031806 00000100:00000010:2.0:1775768105.650561:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d6e40. 00000100:00000001:2.0:1775768105.650565:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1775768105.650568:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768105.650571:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768105.650574:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768105.650579:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768105/real 1775768105] req@ffff9942099fc380 x1862022294274560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768105.650589:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768105.650591:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768105.650593:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768105.650597:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768105.650599:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768105.650602:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768105.650606:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fc380 x1862022294274560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768105.650615:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fc380 x1862022294274560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768105.650622:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768105.650624:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768105.650626:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768105.650628:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768105.650631:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fc380 x1862022294274560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768105.650638:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768105.650643:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fc380 x1862022294274560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768130 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768105.650651:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768105.650654:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768105.650655:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768105.650657:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768105.650659:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768105.650661:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768105.650665:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768105.650669:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768105.650672:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768105.650677:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fc380 x1862022294274560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768130 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768105.650686:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294274560:192.168.202.156@tcp:250:kworker.0 00000100:00000001:2.0:1775768105.650691:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768105.650695:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fed80 x1862022294274688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768105.650702:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768105.650703:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768105.650706:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768105.650708:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768105.650711:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768105.650713:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294274688:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768105.650717:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768105.650719:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768105.650721:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768105.650722:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768105.650726:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1d000. 02000000:00000001:2.0:1775768105.650728:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768105.650732:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768105.650734:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768105.650738:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768105.650741:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294274688, portal 10 00000100:00000001:2.0:1775768105.650744:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768105.650745:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641208192 : -112966068343424 : ffff9942099fed80) 00000100:00000040:2.0:1775768105.650752:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fed80 x1862022294274688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768130 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768105.650759:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768105.650761:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768105.650764:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768105.650766:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294274688, offset 0 00000400:00000200:2.0:1775768105.650770:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768105.650774:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768105.650779:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768105.650782:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768105.650785:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768105.650787:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768105.650790:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768105.650793:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225319:225319:256:4294967295] 192.168.202.156@tcp LPNI seq info [225315:225315:8:4294967295] 00000400:00000200:2.0:1775768105.650797:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768105.650804:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768105.650809:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768105.650812:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768105.650816:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768105.650819:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775768105.650824:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768105.650826:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1775768105.650831:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768105.650833:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768105.650836:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768105.650840:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768105.650843:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768105.650845:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768105.650847:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768105.650851:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fed80 x1862022294274688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768130 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768105.650859:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768105.650863:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fed80 x1862022294274688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768130 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768105.650870:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768105.650890:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768105.650894:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768105.650898:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:2.0:1775768105.650901:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768105.650903:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300030483 00000100:00000001:2.0:1775768105.650906:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300031806 00000100:00000001:2.0:1775768105.650908:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300030483 00000100:00000001:2.0:1775768105.650910:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300031806 00000100:00000001:2.0:1775768105.650913:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768105.650915:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768105.650917:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768105.650922:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768105/real 1775768105] req@ffff9942099fed80 x1862022294274688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768105.650931:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768105.650932:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768105.650934:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768105.650937:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768105.650939:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768105.650941:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768105.650945:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fed80 x1862022294274688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768105.650954:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fed80 x1862022294274688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768105.650961:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768105.650963:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768105.650964:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768105.650966:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768105.650970:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fed80 x1862022294274688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768105.650978:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768105.650982:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fed80 x1862022294274688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768130 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768105.650989:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768105.650992:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768105.650993:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768105.650994:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768105.650996:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768105.651000:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768105.651003:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768105.651006:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768105.651009:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768105.651013:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fed80 x1862022294274688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768130 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768105.651021:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294274688:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768105.651025:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768105.651029:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fd180 x1862022294274816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768105.651035:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768105.651037:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768105.651039:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768105.651041:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768105.651043:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768105.651046:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294274816:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768105.651050:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768105.651053:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768105.651054:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768105.651056:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768105.651060:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1e400. 02000000:00000001:2.0:1775768105.651062:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768105.651066:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768105.651068:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768105.651072:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768105.651075:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294274816, portal 10 00000100:00000001:2.0:1775768105.651078:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768105.651079:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641201024 : -112966068350592 : ffff9942099fd180) 00000100:00000040:2.0:1775768105.651084:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fd180 x1862022294274816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768130 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768105.651092:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768105.651094:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768105.651097:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768105.651099:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294274816, offset 0 00000400:00000200:2.0:1775768105.651103:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768105.651107:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768105.651111:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768105.651114:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768105.651116:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768105.651119:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768105.651121:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768105.651124:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225320:225320:256:4294967295] 192.168.202.156@tcp LPNI seq info [225316:225316:8:4294967295] 00000400:00000200:2.0:1775768105.651129:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768105.651135:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768105.651140:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768105.651144:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768105.651148:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768105.651151:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775768105.651198:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768105.651201:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1775768105.651206:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768105.651209:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768105.651212:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768105.651215:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768105.651218:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768105.651220:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768105.651223:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768105.651227:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fd180 x1862022294274816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768130 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768105.651235:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768105.651239:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fd180 x1862022294274816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768130 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768105.651246:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768105.651247:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768105.651251:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768105.651254:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:2.0:1775768105.651258:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768105.651260:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300030476 00000100:00000001:2.0:1775768105.651263:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300031806 00000100:00000001:2.0:1775768105.651265:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300030476 00000100:00000001:2.0:1775768105.651268:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300031806 00000100:00000001:2.0:1775768105.651270:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768105.651272:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768105.651274:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768105.651278:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768105/real 1775768105] req@ffff9942099fd180 x1862022294274816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768105.651286:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768105.651288:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768105.651289:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768105.651292:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768105.651294:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768105.651297:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768105.651300:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fd180 x1862022294274816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768105.651308:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fd180 x1862022294274816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768105.651316:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768105.651317:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768105.651319:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768105.651321:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768105.651324:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fd180 x1862022294274816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768105.651332:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768105.651335:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fd180 x1862022294274816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768130 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768105.651342:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768105.651345:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768105.651346:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768105.651348:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768105.651350:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768105.651354:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768105.651357:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768105.651360:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768105.651363:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768105.651367:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fd180 x1862022294274816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768130 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768105.651375:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294274816:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768105.651379:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768105.651382:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ff480 x1862022294274944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768105.651389:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768105.651391:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768105.651393:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768105.651395:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768105.651397:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768105.651400:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294274944:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768105.651403:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768105.651405:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768105.651407:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768105.651409:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768105.651413:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f000. 02000000:00000001:2.0:1775768105.651415:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768105.651419:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768105.651422:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768105.651425:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768105.651428:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294274944, portal 4 00000100:00000001:2.0:1775768105.651429:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768105.651431:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641209984 : -112966068341632 : ffff9942099ff480) 00000100:00000040:2.0:1775768105.651436:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ff480 x1862022294274944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768130 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768105.651443:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768105.651445:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768105.651447:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768105.651450:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294274944, offset 0 00000400:00000200:2.0:1775768105.651454:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768105.651458:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768105.651461:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768105.651464:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768105.651466:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768105.651469:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768105.651471:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768105.651475:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225321:225321:256:4294967295] 192.168.202.156@tcp LPNI seq info [225317:225317:8:4294967295] 00000400:00000200:2.0:1775768105.651479:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768105.651485:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768105.651489:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768105.651493:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768105.651497:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768105.651501:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775768105.651505:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768105.651508:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1775768105.651512:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768105.651514:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768105.651517:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768105.651521:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768105.651523:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768105.651525:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768105.651528:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768105.651531:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ff480 x1862022294274944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768130 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768105.651540:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768105.651543:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ff480 x1862022294274944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768130 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768105.651550:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768105.651552:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768105.651555:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768105.651558:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:2.0:1775768105.651562:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768105.651564:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300030499 00000100:00000001:2.0:1775768105.651567:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300031806 00000100:00000001:2.0:1775768105.651569:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300030499 00000100:00000001:2.0:1775768105.651571:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300031806 00000100:00000001:2.0:1775768105.651573:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768105.651576:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768105.651580:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768105.651584:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768105/real 1775768105] req@ffff9942099ff480 x1862022294274944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768105.651592:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768105.651594:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768105.651596:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768105.651599:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768105.651601:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768105.651604:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768105.651608:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ff480 x1862022294274944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768105.651617:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ff480 x1862022294274944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768105.651625:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768105.651627:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768105.651629:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768105.651630:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768105.651634:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ff480 x1862022294274944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768105.651642:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768105.651646:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ff480 x1862022294274944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768130 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768105.651654:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768105.651657:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768105.651658:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768105.651660:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768105.651662:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768105.651665:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768105.651668:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768105.651673:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768105.651675:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768105.651679:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ff480 x1862022294274944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768130 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768105.651688:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294274944:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768105.651692:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768105.651695:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fd880 x1862022294275072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768105.651702:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768105.651704:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768105.651706:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768105.651708:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768105.651710:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768105.651712:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294275072:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768105.651715:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768105.651717:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768105.651719:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768105.651721:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768105.651724:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f800. 02000000:00000001:2.0:1775768105.651727:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768105.651730:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768105.651733:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768105.651737:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768105.651741:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294275072, portal 4 00000100:00000001:2.0:1775768105.651742:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768105.651744:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641202816 : -112966068348800 : ffff9942099fd880) 00000100:00000040:2.0:1775768105.651750:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fd880 x1862022294275072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768130 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768105.651756:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768105.651760:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768105.651763:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768105.651765:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294275072, offset 0 00000400:00000200:2.0:1775768105.651770:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768105.651774:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768105.651779:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768105.651782:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768105.651784:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768105.651786:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768105.651789:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768105.651792:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225322:225322:256:4294967295] 192.168.202.156@tcp LPNI seq info [225318:225318:8:4294967295] 00000400:00000200:2.0:1775768105.651797:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768105.651803:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768105.651808:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768105.651812:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768105.651816:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768105.651819:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775768105.651823:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768105.651825:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1775768105.651830:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768105.651832:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768105.651836:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768105.651840:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768105.651843:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768105.651845:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768105.651848:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768105.651851:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fd880 x1862022294275072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768130 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768105.651860:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768105.651862:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fd880 x1862022294275072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768130 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768105.651869:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768105.652409:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768105.652418:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768105.652422:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:2.0:1775768105.652427:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768105.652430:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300030503 00000100:00000001:2.0:1775768105.652433:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300031806 00000100:00000001:2.0:1775768105.652436:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300030503 00000100:00000001:2.0:1775768105.652438:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300031806 00000100:00000001:2.0:1775768105.652440:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768105.652442:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768105.652445:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768105.652450:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768105/real 1775768105] req@ffff9942099fd880 x1862022294275072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768105.652460:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768105.652461:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768105.652463:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768105.652466:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768105.652467:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768105.652470:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768105.652473:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fd880 x1862022294275072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768105.652482:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fd880 x1862022294275072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768105.652488:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768105.652490:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768105.652491:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768105.652493:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768105.652496:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fd880 x1862022294275072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768105.652502:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768105.652506:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fd880 x1862022294275072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768130 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768105.652513:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768105.652515:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768105.652517:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768105.652518:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768105.652520:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768105.652524:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768105.652527:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768105.652530:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768105.652532:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768105.652536:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fd880 x1862022294275072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768130 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768105.652543:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294275072:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768105.652547:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768105.652550:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768105.652553:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fc380 x1862022294274560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768130 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768105.652560:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768105.652563:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768105.652566:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f400. 02000000:00000001:2.0:1775768105.652569:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768105.652571:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768105.652573:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:2.0:1775768105.652575:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768105.652578:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002b800. 02000000:00000001:2.0:1775768105.652581:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768105.652583:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768105.652585:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fc380. 00000100:00000001:2.0:1775768105.652588:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768105.652589:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768105.652591:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768105.652594:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fed80 x1862022294274688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768130 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768105.652601:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768105.652602:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768105.652603:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1d000. 02000000:00000001:2.0:1775768105.652607:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768105.652608:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768105.652609:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:2.0:1775768105.652611:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768105.652612:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028000. 02000000:00000001:2.0:1775768105.652614:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768105.652615:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768105.652616:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fed80. 00000100:00000001:2.0:1775768105.652618:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768105.652619:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768105.652621:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768105.652624:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fd180 x1862022294274816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768130 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768105.652630:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768105.652631:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768105.652632:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1e400. 02000000:00000001:2.0:1775768105.652635:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768105.652636:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768105.652637:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:2.0:1775768105.652639:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768105.652641:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002bc00. 02000000:00000001:2.0:1775768105.652643:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768105.652644:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768105.652645:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fd180. 00000100:00000001:2.0:1775768105.652648:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768105.652649:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768105.652650:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768105.652653:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ff480 x1862022294274944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768130 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768105.652665:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768105.652667:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768105.652668:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f000. 02000000:00000001:2.0:1775768105.652671:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768105.652672:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768105.652673:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:2.0:1775768105.652675:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768105.652677:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029400. 02000000:00000001:2.0:1775768105.652679:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768105.652680:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768105.652681:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ff480. 00000100:00000001:2.0:1775768105.652683:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768105.652685:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768105.652688:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768105.652692:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fd880 x1862022294275072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768130 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768105.652700:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768105.652702:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768105.652704:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f800. 02000000:00000001:2.0:1775768105.652706:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768105.652707:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768105.652708:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:2.0:1775768105.652711:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768105.652712:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029800. 02000000:00000001:2.0:1775768105.652714:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768105.652716:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768105.652724:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fd880. 00000100:00000001:2.0:1775768105.652727:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768105.652729:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768105.652731:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768105.652739:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768105.652742:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768105.652745:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768105.652747:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768105.713260:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768105.713264:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768105.713268:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768105.713269:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768105.713272:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768105.713274:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768105.713276:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768105.713277:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768105.713279:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768105.713280:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768105.713342:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768105.713346:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768105.713349:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768105.777246:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768105.777251:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768105.777253:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768105.777258:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768105.777261:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768105.777265:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768105.777267:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768105.777269:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768105.777271:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768106.673311:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768106.673314:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768106.673321:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768106.673324:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768106.673334:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:0.0:1775768106.673336:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775768106.673337:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768106.673347:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:2.0:1775768106.673348:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768106.673353:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768106.673353:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768106.673361:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768106.673363:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768106.673365:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768106.673369:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768106.673370:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768106.673374:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768106.673379:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768106.673417:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768106.673421:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768106.673422:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768106.673425:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768106.673427:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768106.673429:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768106.673431:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768106.673432:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768106.673434:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768106.737306:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768106.737316:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768106.737325:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768106.737329:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768106.737332:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768106.737337:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768106.737341:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768106.737344:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768106.737346:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768106.737349:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768106.737354:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768106.737356:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768106.737358:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768106.737359:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768106.737361:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768106.737363:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768106.737365:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768106.737367:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768106.737371:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768106.737372:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768106.737374:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768106.737375:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768106.737376:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768106.737377:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768106.737378:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768106.737380:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768106.737383:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768106.737384:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768106.737385:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768106.737385:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768106.737386:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768106.737387:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768106.737388:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768106.737389:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768106.737449:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768106.737453:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768106.737456:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768106.801322:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768106.801331:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768106.801333:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768106.801341:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768106.801346:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768106.801353:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768106.801357:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768106.801360:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768106.801364:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768106.993345:0:14953:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4300031806, at 4300031808 00000100:00000040:1.0:1775768106.993345:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4300031806, at 4300031808 00000100:00000010:0.0:1775768106.993365:0:14953:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d6a80. 00000100:00000010:1.0:1775768106.993365:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d6e40. 00000100:00000001:0.0:1775768107.697251:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768107.697251:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768107.697255:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768107.697258:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768107.697258:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768107.697261:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768107.697261:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768107.697262:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768107.697265:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768107.697270:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:0.0:1775768107.697271:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775768107.697271:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768107.697274:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768107.697277:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768107.697277:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768107.697278:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768107.697282:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768107.697283:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768107.697283:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768107.697286:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768107.697289:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768107.697290:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768107.697292:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768107.697294:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768107.697297:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768107.697309:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768107.697313:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768107.761247:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768107.761253:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768107.761259:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768107.761260:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768107.761264:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768107.761266:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768107.761269:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768107.761270:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768107.761274:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768107.761274:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768107.761340:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768107.761343:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768107.761347:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768107.825251:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768107.825257:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768107.825260:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768107.825266:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768107.825269:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768107.825274:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768107.825277:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768107.825279:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768107.825283:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768108.721237:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768108.721237:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768108.721237:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768108.721243:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768108.721243:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768108.721244:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768108.721246:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768108.721247:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768108.721247:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768108.721253:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:2.0:1775768108.721254:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775768108.721254:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775768108.721260:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768108.721260:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768108.721261:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768108.721265:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768108.721265:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768108.721266:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768108.721268:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768108.721272:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768108.721274:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768108.721274:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768108.721276:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768108.721279:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768108.721280:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768108.721283:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768108.721286:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768108.785247:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768108.785252:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768108.785257:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768108.785259:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768108.785262:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768108.785263:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768108.785265:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768108.785266:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768108.785276:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768108.785277:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768108.785323:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768108.785326:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768108.785329:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768108.849267:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768108.849272:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768108.849273:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768108.849278:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768108.849281:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768108.849284:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768108.849287:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768108.849289:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768108.849291:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768109.745240:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768109.745242:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768109.745245:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768109.745246:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768109.745247:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768109.745248:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768109.745252:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775768109.745253:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775768109.745257:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768109.745259:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768109.745260:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768109.745262:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768109.745263:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768109.745266:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768109.745269:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768109.745269:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768109.745272:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768109.745276:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768109.745291:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768109.745296:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768109.745298:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768109.745302:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:0.0:1775768109.745306:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768109.745309:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768109.745311:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768109.745313:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768109.745315:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768109.809246:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768109.809251:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768109.809257:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768109.809259:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768109.809262:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768109.809264:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768109.809268:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768109.809269:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768109.809273:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768109.809274:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768109.809333:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768109.809337:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768109.809339:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768109.873243:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768109.873250:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768109.873252:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768109.873258:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768109.873262:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768109.873266:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768109.873269:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768109.873272:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768109.873275:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768110.769304:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768110.769305:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768110.769305:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768110.769311:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768110.769312:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768110.769316:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768110.769317:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768110.769317:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768110.769326:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768110.769327:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:2.0:1775768110.769327:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768110.769334:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768110.769338:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768110.769342:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775768110.769342:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768110.769343:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768110.769349:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768110.769353:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768110.769354:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768110.769360:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768110.769362:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768110.769363:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1775768110.769369:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775768110.769372:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768110.769380:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768110.769380:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:1.0:1775768110.769382:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775768110.769386:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768110.769388:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1775768110.769390:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768110.769393:0:36:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768110.769399:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775768110.769412:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5363 00000400:00000200:1.0:1775768110.769433:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768110.769443:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775768110.769447:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775768110.769450:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768110.769454:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768110.769457:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768110.769459:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775768110.769464:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775768110.769467:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:1.0:1775768110.769470:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775768110.769473:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:1.0:1775768110.769476:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:1.0:1775768110.769479:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:1.0:1775768110.769482:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768110.769485:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768110.769487:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768110.769489:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775768110.769492:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:1.0:1775768110.769496:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775768110.769499:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768110.769502:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775768110.769507:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768110.769510:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768110.769512:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775768110.769515:0:36:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775768110.769528:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207889f80. 00000020:00000040:1.0:1775768110.769533:0:36:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:1.0:1775768110.769537:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768110.769539:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768110.769543:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775768110.769546:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775768110.769552:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775768110.769564:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543fc00. 00000100:00000001:1.0:1775768110.769572:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768110.769583:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207889f80 x1862022294275200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775768110.769594:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775768110.769600:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768110.769605:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ed2274f4] to pc [ptlrpcd_rcv+-1] req@ffff994207889f80 x1862022294275200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768110.769639:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768110.769642:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768110.769646:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775768110.769648:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775768110.769650:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768110.769655:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775768110.769660:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5363 00000400:00000200:1.0:1775768110.769666:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768110.769668:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:1.0:1775768110.769673:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:2.0:1775768110.769674:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768110.769677:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00080000:1.0:1775768110.769681:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775768110.769688:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000040:2.0:1775768110.769692:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207889f80 x1862022294275200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768110.769694:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768110.769701:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768110.769714:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768110.769716:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768110.769719:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 02000000:00000001:2.0:1775768110.769720:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:1.0:1775768110.769725:0:36:0:(genops.c:811:class_conn2export()) Process entered 02000000:00000001:2.0:1775768110.769726:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:1.0:1775768110.769727:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:1.0:1775768110.769735:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000001:2.0:1775768110.769736:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1775768110.769740:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:1.0:1775768110.769745:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00100000:2.0:1775768110.769746:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207889f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294275200:192.168.202.156@tcp:250:kworker.0 00000020:00000001:1.0:1775768110.769753:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000001:2.0:1775768110.769757:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000040:1.0:1775768110.769762:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 02000000:00000001:2.0:1775768110.769763:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768110.769767:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768110.769768:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1775768110.769770:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:1.0:1775768110.769775:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768110.769778:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 02000000:00000010:2.0:1775768110.769781:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f800. 00000020:00000040:1.0:1775768110.769783:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 02000000:00000001:2.0:1775768110.769784:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768110.769796:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000400:00000200:2.0:1775768110.769804:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768110.769808:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775768110.769811:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 02000000:00000001:1.0:1775768110.769815:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:2.0:1775768110.769819:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 02000000:00000001:1.0:1775768110.769838:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768110.769843:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1775768110.769844:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294275200, portal 25 00000100:00000001:1.0:1775768110.769846:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775768110.769848:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768110.769850:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606126464 : -112966103425152 : ffff994207889f80) 00000008:00100000:1.0:1775768110.769853:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00000040:2.0:1775768110.769858:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207889f80 x1862022294275200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768135 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:1.0:1775768110.769859:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768110.769869:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775768110.769878:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff99420788b100. 00000020:00000040:1.0:1775768110.769883:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:1.0:1775768110.769893:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768110.769896:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768110.769896:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1775768110.769903:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775768110.769907:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:2.0:1775768110.769907:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 02000000:00000001:1.0:1775768110.769912:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775768110.769915:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 02000000:00000010:1.0:1775768110.769920:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543dc00. 00000100:00000200:2.0:1775768110.769922:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294275200, offset 0 00000100:00000001:1.0:1775768110.769926:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768110.769939:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff99420788b100 x1862022294275328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1775768110.769939:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00000001:1.0:1775768110.769953:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:2.0:1775768110.769957:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768110.769959:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768110.769971:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e26cf977] to pc [ptlrpcd_rcv+-1] req@ffff99420788b100 x1862022294275328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1775768110.769977:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768110.769988:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768110.769994:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000100:00000001:1.0:1775768110.769997:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768110.770000:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000100:00080000:1.0:1775768110.770005:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:2.0:1775768110.770005:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768110.770014:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225323:225323:256:4294967295] 192.168.202.156@tcp LPNI seq info [225319:225319:8:4294967295] 00000100:00080000:1.0:1775768110.770016:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775768110.770025:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:2.0:1775768110.770027:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00080000:1.0:1775768110.770031:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768110.770037:0:36:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:2.0:1775768110.770043:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000100:00080000:1.0:1775768110.770049:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5363 00000800:00000200:2.0:1775768110.770054:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768110.770061:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000400:00000200:1.0:1775768110.770062:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768110.770073:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000800:00000200:2.0:1775768110.770073:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00080000:1.0:1775768110.770085:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000800:00000200:2.0:1775768110.770086:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000100:00000040:1.0:1775768110.770097:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000800:00000200:2.1:1775768110.770098:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:1.0:1775768110.770107:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:2.1:1775768110.770107:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000100:00000040:1.0:1775768110.770118:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000800:00000100:2.0:1775768110.770119:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768110.770122:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000100:00000001:1.0:1775768110.770128:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000200:2.0:1775768110.770131:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000001:1.0:1775768110.770133:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775768110.770137:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:1.0:1775768110.770142:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:2.0:1775768110.770143:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000020:00000040:1.0:1775768110.770146:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:1.0:1775768110.770150:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000400:00000200:2.0:1775768110.770151:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768110.770188:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768110.770195:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:1.0:1775768110.770201:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000200:2.0:1775768110.770203:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207889f80 x1862022294275200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768135 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775768110.770213:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768110.770223:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768110.770228:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768110.770228:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1775768110.770238:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000040:2.0:1775768110.770238:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207889f80 x1862022294275200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768135 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1775768110.770250:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00000001:2.0:1775768110.770260:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768110.770263:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775768110.770264:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768110.770269:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768110.770273:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1775768110.770274:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768110.770278:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768110.770285:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768110.770286:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768110.770292:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775768110.770293:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4300036926 00000008:00100000:1.0:1775768110.770296:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775768110.770303:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775768110.770304:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d63c0. 00000100:00000001:1.0:1775768110.770308:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768110.770314:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000010:1.0:1775768110.770315:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207888700. 00000020:00000040:1.0:1775768110.770321:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000100:00000001:2.0:1775768110.770324:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4300036926 02000000:00000001:1.0:1775768110.770325:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768110.770329:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775768110.770331:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d68a0. 00000100:00000001:1.0:1775768110.770333:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1775768110.770336:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 02000000:00000001:1.0:1775768110.770337:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775768110.770342:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768110.770344:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768110.770353:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768110.770355:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 02000000:00000010:1.0:1775768110.770356:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543d800. 00000100:00100000:2.0:1775768110.770363:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768110/real 1775768110] req@ffff994207889f80 x1862022294275200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768110.770366:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1775768110.770381:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768110.770382:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768110.770384:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768110.770387:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768110.770388:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768110.770391:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:1.0:1775768110.770400:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207888700 x1862022294275456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:2.0:1775768110.770401:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207889f80 x1862022294275200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768110.770458:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000200:2.0:1775768110.770460:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207889f80 x1862022294275200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768110.770464:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768110.770476:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768110.770478:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768110.770479:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009f2ba301] to pc [ptlrpcd_rcv+-1] req@ffff994207888700 x1862022294275456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768110.770482:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768110.770485:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768110.770495:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207889f80 x1862022294275200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768110.770523:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768110.770525:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1775768110.770528:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768110.770535:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000040:2.0:1775768110.770536:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207889f80 x1862022294275200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768135 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768110.770538:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775768110.770541:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768110.770549:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775768110.770556:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5363 00000400:00000200:1.0:1775768110.770573:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768110.770578:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000400:00000200:1.0:1775768110.770583:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:2.0:1775768110.770584:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1775768110.770591:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1775768110.770592:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768110.770595:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768110.770602:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000040:1.0:1775768110.770606:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768110.770606:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768110.770616:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00080000:2.0:1775768110.770616:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000040:1.0:1775768110.770625:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00080000:2.0:1775768110.770625:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768110.770633:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:1.0:1775768110.770634:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000040:2.0:1775768110.770639:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207889f80 x1862022294275200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768135 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1775768110.770640:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775768110.770644:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:1.0:1775768110.770652:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775768110.770657:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000100:00100000:2.0:1775768110.770663:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207889f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294275200:192.168.202.156@tcp:250:kworker.0 00000020:00000001:1.0:1775768110.770664:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:2.0:1775768110.770672:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1775768110.770673:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:2.0:1775768110.770677:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768110.770684:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207889f80 x1862022294275200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768135 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775768110.770685:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768110.770694:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775768110.770703:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768110.770707:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768110.770710:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f800. 02000000:00000001:2.0:1775768110.770714:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768110.770717:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768110.770719:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:2.0:1775768110.770721:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768110.770724:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543fc00. 02000000:00000001:2.0:1775768110.770728:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000040:1.0:1775768110.770730:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000001:2.0:1775768110.770732:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1775768110.770736:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000010:2.0:1775768110.770737:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207889f80. 00000020:00000040:1.0:1775768110.770746:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00000001:2.0:1775768110.770748:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768110.770749:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1775768110.770753:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775768110.770754:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768110.770762:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768110.770766:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:1.0:1775768110.770767:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1775768110.770769:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768110.770772:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1775768110.770773:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768110.770775:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768110.770778:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1775768110.770783:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768110.770790:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000040:2.0:1775768110.770791:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff99420788b100 x1862022294275328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1775768110.770795:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:1.0:1775768110.770804:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768110.770810:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775768110.770818:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207889880. 00000100:00000001:2.0:1775768110.770820:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768110.770837:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1775768110.770841:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:2.0:1775768110.770842:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768110.770845:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768110.770848:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775768110.770850:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768110.770851:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768110.770855:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:2.0:1775768110.770861:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff99420788b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294275328:192.168.202.156@tcp:38:kworker.0 02000000:00000001:1.0:1775768110.770862:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775768110.770867:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768110.770870:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000010:1.0:1775768110.770874:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543f800. 02000000:00000001:2.0:1775768110.770874:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768110.770878:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768110.770881:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768110.770883:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768110.770889:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f800. 00000100:00100000:1.0:1775768110.770893:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207889880 x1862022294275584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1775768110.770895:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768110.770911:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768110.770919:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000010:2.0:1775768110.770920:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000100:00000001:1.0:1775768110.770925:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775768110.770928:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000040:1.0:1775768110.770933:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f11856b8] to pc [ptlrpcd_rcv+-1] req@ffff994207889880 x1862022294275584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768110.770935:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294275328, portal 10 00000100:00000001:2.0:1775768110.770942:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768110.770945:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606130944 : -112966103420672 : ffff99420788b100) 00000100:00000001:1.0:1775768110.770958:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768110.770959:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff99420788b100 x1862022294275328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768135 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768110.770963:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768110.770980:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775768110.770986:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775768110.770989:0:36:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775768110.770995:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1775768110.770996:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775768110.771003:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5363 00000100:00000040:2.0:1775768110.771005:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768110.771012:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000400:00000200:1.0:1775768110.771015:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000200:2.0:1775768110.771018:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294275328, offset 0 00000400:00000200:1.0:1775768110.771023:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775768110.771027:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1775768110.771033:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00000040:1.0:1775768110.771038:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000200:2.0:1775768110.771041:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768110.771050:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1775768110.771054:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00000040:1.0:1775768110.771059:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000200:2.0:1775768110.771062:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1775768110.771068:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000200:2.0:1775768110.771072:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000020:00000001:1.0:1775768110.771078:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000400:00000200:2.0:1775768110.771084:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000020:00000040:1.0:1775768110.771085:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:1.0:1775768110.771090:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:2.0:1775768110.771091:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000020:00000040:1.0:1775768110.771095:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000400:00000200:2.0:1775768110.771099:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225324:225324:256:4294967295] 192.168.202.156@tcp LPNI seq info [225320:225320:8:4294967295] 00000020:00000001:1.0:1775768110.771107:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000400:00000200:2.0:1775768110.771111:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:1.0:1775768110.771117:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000400:00000200:2.0:1775768110.771127:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000100:00000040:1.0:1775768110.771131:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000800:00000200:2.0:1775768110.771136:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00000001:1.0:1775768110.771138:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000010:2.0:1775768110.771142:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000100:00000040:1.0:1775768110.771144:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768110.771190:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000800:00000200:2.0:1775768110.771191:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000040:1.0:1775768110.771199:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000800:00000200:2.0:1775768110.771200:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000100:00080000:1.0:1775768110.771210:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775768110.771218:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775768110.771218:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768110.771225:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 02000000:00000001:1.0:1775768110.771227:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000800:00000100:2.0:1775768110.771231:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 02000000:00000001:1.0:1775768110.771233:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000010:2.0:1775768110.771234:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 02000000:00000001:1.0:1775768110.771241:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768110.771241:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:1.0:1775768110.771251:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:2.0:1775768110.771254:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000008:00100000:1.0:1775768110.771257:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000400:00000200:2.0:1775768110.771258:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768110.771262:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000008:00000001:1.0:1775768110.771267:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768110.771268:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1775768110.771272:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1775768110.771278:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff99420788b100 x1862022294275328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768135 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1775768110.771281:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207888e00. 00000020:00000040:1.0:1775768110.771288:0:36:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:1.0:1775768110.771295:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768110.771297:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768110.771301:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1775768110.771301:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 02000000:00000001:1.0:1775768110.771305:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775768110.771310:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768110.771311:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff99420788b100 x1862022294275328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768135 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1775768110.771320:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543c800. 00000100:00000001:2.0:1775768110.771326:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768110.771328:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768110.771329:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768110.771337:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768110.771342:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207888e00 x1862022294275712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:2.0:1775768110.771343:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:2.0:1775768110.771358:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768110.771363:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300035604 00000100:00000001:2.0:1775768110.771369:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300036926 00000100:00000001:2.0:1775768110.771377:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300035604 00000100:00000001:1.0:1775768110.771379:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775768110.771380:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300036926 00000100:00000001:1.0:1775768110.771387:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768110.771390:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1775768110.771398:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001812f966] to pc [ptlrpcd_rcv+-1] req@ffff994207888e00 x1862022294275712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768110.771398:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768110.771407:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768110.771418:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768110/real 1775768110] req@ffff99420788b100 x1862022294275328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768110.771420:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768110.771430:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5373) 00000400:00000001:2.0:1775768110.771441:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768110.771443:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768110.771446:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768110.771449:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768110.771451:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768110.771453:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768110.771457:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff99420788b100 x1862022294275328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768110.771464:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff99420788b100 x1862022294275328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768110.771469:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768110.771470:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768110.771471:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768110.771472:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768110.771474:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff99420788b100 x1862022294275328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768110.771478:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768110.771481:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff99420788b100 x1862022294275328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768135 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768110.771486:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768110.771487:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768110.771489:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768110.771490:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768110.771491:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768110.771493:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768110.771495:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768110.771498:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768110.771499:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768110.771502:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff99420788b100 x1862022294275328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768135 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768110.771509:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff99420788b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294275328:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768110.771514:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768110.771518:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207888700 x1862022294275456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768110.771524:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768110.771526:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768110.771528:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768110.771530:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768110.771533:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768110.771536:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207888700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294275456:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768110.771540:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768110.771543:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768110.771544:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768110.771546:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768110.771551:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f000. 02000000:00000001:2.0:1775768110.771553:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768110.771557:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768110.771560:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768110.771563:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768110.771566:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294275456, portal 10 00000100:00000001:2.0:1775768110.771567:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768110.771568:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606120192 : -112966103431424 : ffff994207888700) 00000100:00000040:2.0:1775768110.771573:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207888700 x1862022294275456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768135 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768110.771577:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768110.771579:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768110.771580:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768110.771582:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294275456, offset 0 00000400:00000200:2.0:1775768110.771585:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768110.771588:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768110.771590:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768110.771592:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768110.771594:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768110.771596:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768110.771597:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768110.771600:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225325:225325:256:4294967295] 192.168.202.156@tcp LPNI seq info [225321:225321:8:4294967295] 00000400:00000200:2.0:1775768110.771603:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768110.771607:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768110.771610:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768110.771612:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768110.771615:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768110.771617:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775768110.771620:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768110.771622:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1775768110.771624:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768110.771626:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768110.771628:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768110.771630:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768110.771632:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768110.771633:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768110.771635:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768110.771637:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207888700 x1862022294275456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768135 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768110.771646:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768110.771648:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207888700 x1862022294275456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768135 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768110.771652:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768110.771653:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768110.771655:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768110.771657:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:2.0:1775768110.771659:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768110.771660:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300035596 00000100:00000001:2.0:1775768110.771661:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300036926 00000100:00000001:2.0:1775768110.771663:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300035596 00000100:00000001:2.0:1775768110.771664:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300036926 00000100:00000001:2.0:1775768110.771666:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768110.771668:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768110.771669:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768110.771671:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768110/real 1775768110] req@ffff994207888700 x1862022294275456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768110.771675:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768110.771676:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768110.771677:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768110.771678:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768110.771679:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768110.771680:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768110.771682:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207888700 x1862022294275456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768110.771687:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207888700 x1862022294275456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768110.771692:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768110.771694:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768110.771696:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768110.771697:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768110.771701:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207888700 x1862022294275456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768110.771708:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768110.771711:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207888700 x1862022294275456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768135 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768110.771719:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768110.771721:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768110.771723:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768110.771724:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768110.771726:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768110.771730:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768110.771732:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768110.771736:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768110.771739:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768110.771743:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207888700 x1862022294275456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768135 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768110.771752:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207888700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294275456:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768110.771756:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768110.771757:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768110.771760:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff99420788b100 x1862022294275328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768135 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768110.771764:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768110.771765:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768110.771767:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f800. 02000000:00000001:2.0:1775768110.771769:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768110.771770:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768110.771771:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:2.0:1775768110.771773:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768110.771774:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543dc00. 02000000:00000001:2.0:1775768110.771776:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768110.771777:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768110.771778:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff99420788b100. 00000100:00000001:2.0:1775768110.771780:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768110.771781:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768110.771782:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768110.771784:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207888700 x1862022294275456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768135 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768110.771788:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768110.771789:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768110.771790:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f000. 02000000:00000001:2.0:1775768110.771791:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768110.771791:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768110.771792:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:2.0:1775768110.771793:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768110.771794:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543d800. 02000000:00000001:2.0:1775768110.771795:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768110.771796:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768110.771797:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207888700. 00000100:00000001:2.0:1775768110.771798:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768110.771799:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768110.771800:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768110.771802:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768110.771803:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768110.771805:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768110.771806:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768110.771807:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768110.771809:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207889880 x1862022294275584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768110.771814:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768110.771815:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768110.771816:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768110.771817:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768110.771819:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768110.771821:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207889880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294275584:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768110.771838:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768110.771840:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768110.771841:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768110.771843:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768110.771845:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f000. 02000000:00000001:2.0:1775768110.771846:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768110.771849:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768110.771851:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768110.771854:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768110.771856:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294275584, portal 4 00000100:00000001:2.0:1775768110.771857:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768110.771859:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606124672 : -112966103426944 : ffff994207889880) 00000100:00000040:2.0:1775768110.771862:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207889880 x1862022294275584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768135 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768110.771867:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768110.771868:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768110.771870:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768110.771871:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294275584, offset 0 00000400:00000200:2.0:1775768110.771874:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768110.771878:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768110.771881:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768110.771884:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768110.771887:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768110.771889:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768110.771891:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768110.771894:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225326:225326:256:4294967295] 192.168.202.156@tcp LPNI seq info [225322:225322:8:4294967295] 00000400:00000200:2.0:1775768110.771898:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768110.771905:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768110.771909:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768110.771914:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768110.771918:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768110.771921:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775768110.771925:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768110.771928:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1775768110.771932:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768110.771934:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768110.771937:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768110.771941:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768110.771943:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768110.771945:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768110.771948:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768110.771952:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207889880 x1862022294275584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768135 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768110.771960:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768110.771962:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207889880 x1862022294275584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768135 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768110.771967:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768110.771968:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768110.771970:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768110.771973:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:2.0:1775768110.771975:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768110.771977:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300035619 00000100:00000001:2.0:1775768110.771979:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300036926 00000100:00000001:2.0:1775768110.771980:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300035619 00000100:00000001:2.0:1775768110.771982:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300036926 00000100:00000001:2.0:1775768110.771983:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768110.771985:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768110.771986:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768110.771988:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768110/real 1775768110] req@ffff994207889880 x1862022294275584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768110.771998:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768110.771999:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768110.772000:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768110.772002:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768110.772003:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768110.772004:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768110.772006:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207889880 x1862022294275584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768110.772011:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207889880 x1862022294275584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768110.772016:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768110.772017:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768110.772018:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768110.772019:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768110.772021:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207889880 x1862022294275584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768110.772025:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768110.772027:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207889880 x1862022294275584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768135 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768110.772031:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768110.772032:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768110.772033:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768110.772034:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768110.772036:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768110.772040:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768110.772042:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768110.772046:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768110.772048:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768110.772052:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207889880 x1862022294275584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768135 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768110.772060:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207889880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294275584:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768110.772065:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768110.772069:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207888e00 x1862022294275712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768110.772076:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768110.772078:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768110.772080:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768110.772082:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768110.772084:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768110.772088:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207888e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294275712:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768110.772092:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768110.772094:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768110.772096:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768110.772098:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768110.772101:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f800. 02000000:00000001:2.0:1775768110.772103:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768110.772106:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768110.772109:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768110.772113:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768110.772116:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294275712, portal 4 00000100:00000001:2.0:1775768110.772119:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768110.772120:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606121984 : -112966103429632 : ffff994207888e00) 00000100:00000040:2.0:1775768110.772125:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207888e00 x1862022294275712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768135 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768110.772132:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768110.772135:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768110.772138:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768110.772140:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294275712, offset 0 00000400:00000200:2.0:1775768110.772145:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768110.772149:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768110.772153:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768110.772179:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768110.772182:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768110.772184:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768110.772186:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768110.772188:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225327:225327:256:4294967295] 192.168.202.156@tcp LPNI seq info [225323:225323:8:4294967295] 00000400:00000200:2.0:1775768110.772191:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768110.772197:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768110.772202:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768110.772206:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768110.772210:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768110.772213:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775768110.772217:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768110.772220:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1775768110.772224:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768110.772226:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768110.772229:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768110.772232:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768110.772236:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768110.772238:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768110.772240:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768110.772244:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207888e00 x1862022294275712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768135 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768110.772253:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768110.772256:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207888e00 x1862022294275712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768135 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768110.772261:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768110.772262:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768110.772265:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768110.772268:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:2.0:1775768110.772271:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768110.772274:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300035623 00000100:00000001:2.0:1775768110.772277:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300036926 00000100:00000001:2.0:1775768110.772279:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300035623 00000100:00000001:2.0:1775768110.772281:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300036926 00000100:00000001:2.0:1775768110.772283:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768110.772285:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768110.772287:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768110.772291:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768110/real 1775768110] req@ffff994207888e00 x1862022294275712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768110.772300:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768110.772302:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768110.772304:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768110.772306:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768110.772308:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768110.772311:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768110.772314:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207888e00 x1862022294275712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768110.772324:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207888e00 x1862022294275712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768110.772329:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768110.772330:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768110.772331:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768110.772333:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768110.772335:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207888e00 x1862022294275712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768110.772340:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768110.772342:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207888e00 x1862022294275712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768135 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768110.772346:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768110.772349:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768110.772350:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768110.772351:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768110.772352:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768110.772355:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768110.772357:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768110.772359:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768110.772360:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768110.772363:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207888e00 x1862022294275712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768135 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768110.772368:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207888e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294275712:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768110.772371:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768110.772372:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768110.772374:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207889880 x1862022294275584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768135 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768110.772378:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768110.772379:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768110.772380:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f000. 02000000:00000001:2.0:1775768110.772382:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768110.772383:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768110.772384:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:2.0:1775768110.772387:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768110.772388:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543f800. 02000000:00000001:2.0:1775768110.772391:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768110.772393:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768110.772394:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207889880. 00000100:00000001:2.0:1775768110.772397:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768110.772398:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768110.772400:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768110.772403:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207888e00 x1862022294275712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768135 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768110.772416:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768110.772418:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768110.772420:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f800. 02000000:00000001:2.0:1775768110.772422:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768110.772423:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768110.772425:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:2.0:1775768110.772427:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768110.772429:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543c800. 02000000:00000001:2.0:1775768110.772431:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768110.772432:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768110.772433:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207888e00. 00000100:00000001:2.0:1775768110.772436:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768110.772437:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768110.772439:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768110.772443:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768110.772444:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768110.772446:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768110.772448:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768110.833314:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768110.833321:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768110.833326:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768110.833327:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768110.833331:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768110.833332:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768110.833335:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768110.833336:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768110.833339:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768110.833340:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768110.833437:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768110.833441:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768110.833444:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768110.897320:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768110.897329:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768110.897332:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768110.897339:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768110.897345:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768110.897351:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768110.897355:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768110.897358:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768110.897362:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768111.793275:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768111.793275:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768111.793275:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768111.793284:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768111.793285:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768111.793285:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768111.793287:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768111.793288:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768111.793289:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768111.793296:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:0.0:1775768111.793297:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775768111.793297:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768111.793302:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768111.793303:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768111.793304:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768111.793307:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768111.793308:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768111.793309:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768111.793313:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768111.793316:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768111.793319:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768111.793320:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768111.793323:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768111.793326:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768111.793326:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768111.793329:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768111.793333:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768111.857254:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768111.857263:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768111.857265:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768111.857269:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768111.857275:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768111.857277:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768111.857279:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768111.857281:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768111.857284:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768111.857285:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768111.857289:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768111.857289:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768111.857292:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768111.857293:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768111.857296:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768111.857297:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768111.857356:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768111.857360:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768111.857363:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768111.921298:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768111.921305:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768111.921307:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768111.921313:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768111.921317:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768111.921322:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768111.921325:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768111.921328:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768111.921330:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768112.113287:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4300036926, at 4300036928 00000100:00000040:0.0:1775768112.113297:0:14953:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4300036926, at 4300036928 00000100:00000010:1.0:1775768112.113308:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d68a0. 00000100:00000010:0.0:1775768112.113311:0:14953:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d63c0. 00000100:00000001:3.0:1775768112.817312:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768112.817320:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768112.817323:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768112.817325:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768112.817326:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:3.0:1775768112.817328:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768112.817335:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768112.817336:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768112.817337:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768112.817342:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768112.817343:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768112.817343:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768112.817355:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768112.817355:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00100000:2.0:1775768112.817356:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775768112.817361:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768112.817364:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768112.817366:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768112.817368:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768112.817372:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768112.817373:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768112.817375:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768112.817377:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768112.817381:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768112.817382:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768112.817389:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768112.817395:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768112.881277:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768112.881284:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768112.881288:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768112.881289:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768112.881293:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768112.881293:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768112.881296:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768112.881297:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768112.881300:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768112.881301:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768112.881373:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768112.881380:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768112.881384:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768112.945285:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768112.945293:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768112.945295:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768112.945303:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768112.945308:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768112.945314:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768112.945318:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768112.945321:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768112.945325:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768113.841254:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768113.841255:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768113.841263:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768113.841264:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768113.841266:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768113.841267:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768113.841272:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775768113.841273:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775768113.841279:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768113.841280:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768113.841284:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768113.841284:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768113.841290:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768113.841293:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768113.841295:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768113.841297:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768113.841298:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768113.841299:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768113.841303:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768113.841303:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768113.841309:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768113.841310:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:0.0:1775768113.841316:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768113.841319:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768113.841322:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768113.841325:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768113.841327:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768113.905261:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768113.905266:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768113.905270:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768113.905271:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768113.905273:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768113.905274:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768113.905278:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768113.905279:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768113.905281:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768113.905282:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768113.905342:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768113.905346:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768113.905349:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768113.969245:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768113.969250:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768113.969252:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768113.969256:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768113.969259:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768113.969262:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768113.969264:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768113.969266:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768113.969268:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768114.865240:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768114.865240:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768114.865245:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768114.865246:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768114.865247:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768114.865249:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768114.865250:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768114.865250:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768114.865251:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768114.865257:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775768114.865257:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:0.0:1775768114.865258:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768114.865262:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768114.865264:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768114.865264:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768114.865266:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768114.865268:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768114.865269:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768114.865269:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768114.865272:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768114.865275:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768114.865276:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768114.865278:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768114.865281:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768114.865281:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768114.865284:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768114.865286:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768114.929238:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768114.929242:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768114.929247:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768114.929249:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768114.929251:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768114.929253:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768114.929255:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768114.929256:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768114.929259:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768114.929260:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768114.929313:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768114.929317:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768114.929319:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768114.993281:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768114.993286:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768114.993288:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768114.993294:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768114.993297:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768114.993300:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768114.993302:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768114.993305:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768114.993307:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768115.889307:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768115.889311:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768115.889312:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:3.0:1775768115.889312:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775768115.889314:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768115.889321:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768115.889323:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768115.889324:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768115.889329:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768115.889330:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768115.889330:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:0.0:1775768115.889333:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768115.889335:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:1.0:1775768115.889337:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775768115.889338:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00080000:3.0:1775768115.889340:0:36:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768115.889345:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768115.889346:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768115.889347:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:0.0:1775768115.889351:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768115.889354:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768115.889354:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768115.889357:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775768115.889357:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5368 00000100:00000001:2.0:1775768115.889362:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768115.889367:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768115.889372:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768115.889372:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775768115.889376:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768115.889377:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768115.889382:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768115.889383:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000400:00000200:3.0:1775768115.889385:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:0.0:1775768115.889391:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775768115.889392:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1775768115.889401:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768115.889401:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768115.889410:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775768115.889421:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768115.889425:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775768115.889429:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775768115.889431:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:3.0:1775768115.889434:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775768115.889437:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:3.0:1775768115.889439:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000080:00000001:2.0:1775768115.889440:0:13471:0:(rw.c:1481:ll_writepages()) Process entered 00000020:00000001:3.0:1775768115.889441:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000080:00000001:2.0:1775768115.889447:0:13471:0:(file.c:5250:cl_sync_file_range()) Process entered 00000100:00000040:3.0:1775768115.889447:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768115.889451:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:2.0:1775768115.889454:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00000040:3.0:1775768115.889455:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768115.889458:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775768115.889460:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:3.0:1775768115.889463:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775768115.889465:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1775768115.889467:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994210b32630. 02000000:00000001:3.0:1775768115.889468:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775768115.889472:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775768115.889475:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1775768115.889476:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9942060d4800. 00000100:00000001:3.0:1775768115.889477:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775768115.889480:0:36:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1775768115.889488:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff9942060d5a00. 00000100:00000010:3.0:1775768115.889488:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8e00. 00000020:00000040:3.0:1775768115.889494:0:36:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 00000020:00000010:2.0:1775768115.889496:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942182d6a20. 02000000:00000001:3.0:1775768115.889498:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775768115.889501:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768115.889503:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000008:00000010:2.0:1775768115.889506:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207ccc2f0. 02000000:00000001:3.0:1775768115.889507:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775768115.889512:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:2.0:1775768115.889515:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994220156e40. 02000000:00000010:3.0:1775768115.889522:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0000. 00008000:00000010:2.0:1775768115.889524:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942077a8e20. 00000100:00000001:3.0:1775768115.889530:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:2.0:1775768115.889534:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff9942087472a0. 00000080:00000010:2.0:1775768115.889541:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b8000. 00000100:00100000:3.0:1775768115.889541:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8e00 x1862022294275840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000010:2.0:1775768115.889548:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9942060d5200. 00000008:00000010:2.0:1775768115.889555:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942077cd600. 00000100:00000001:3.0:1775768115.889559:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00020000:00000010:2.0:1775768115.889561:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff99420876d680. 00000100:00000001:3.0:1775768115.889565:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:2.0:1775768115.889568:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076cf60. 00000100:00000040:3.0:1775768115.889572:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000864a6866] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8e00 x1862022294275840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000010:2.0:1775768115.889575:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff994205066870. 00000020:00000001:2.0:1775768115.889581:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107759908408 : -112965949643208 : ffff994210b32638) 00000020:00001000:2.0:1775768115.889587:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff994210b32630 00000020:00000001:2.0:1775768115.889594:0:13471:0:(cl_object.c:151:cl_object_top()) ffff99420791a5f8 -> ffff99420791a5f8 00000020:00000001:2.0:1775768115.889598:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:2.0:1775768115.889601:0:13471:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000100:00000001:3.0:1775768115.889604:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000080:00200000:2.0:1775768115.889605:0:13471:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x10176:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000100:00080000:3.0:1775768115.889610:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000001:2.0:1775768115.889612:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775768115.889615:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00080000:3.0:1775768115.889617:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000080:00000001:2.0:1775768115.889618:0:13471:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768115.889620:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775768115.889624:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00020000:00000002:2.0:1775768115.889625:0:13471:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x10176:0x0]io ffff9942077b8060 type 6 ignore/verify layout 1/0 00000100:00000001:3.0:1775768115.889630:0:36:0:(import.c:484:import_select_connection()) Process entered 00020000:00000002:2.0:1775768115.889632:0:13471:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207dfb6c8) owner 0000000000000000/ffff994220080000 00000100:00080000:3.0:1775768115.889637:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5368 00000100:00000001:1.0:1775768115.889639:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00020000:00000001:2.0:1775768115.889639:0:13471:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:2.0:1775768115.889644:0:13471:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00000100:00000001:1.0:1775768115.889645:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000400:00000200:3.0:1775768115.889645:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768115.889651:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00020000:00000001:2.0:1775768115.889651:0:13471:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00000400:00000200:3.0:1775768115.889651:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00020000:00000001:2.0:1775768115.889655:0:13471:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768115.889656:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00020000:00000001:2.0:1775768115.889660:0:13471:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1775768115.889662:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000040:1.0:1775768115.889663:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a8e00 x1862022294275840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:2.0:1775768115.889664:0:13471:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00000100:00000001:3.0:1775768115.889668:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00020000:00000001:2.0:1775768115.889669:0:13471:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768115.889673:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000001:2.0:1775768115.889674:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775768115.889677:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00000001:3.0:1775768115.889679:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00020000:00000001:2.0:1775768115.889681:0:13471:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00000020:00000001:3.0:1775768115.889684:0:36:0:(genops.c:811:class_conn2export()) Process entered 00020000:00000002:2.0:1775768115.889686:0:13471:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207dfb6c8) owner 0000000000000000/ffff994220080000 00000020:00000040:3.0:1775768115.889688:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:3.0:1775768115.889691:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000001:2.0:1775768115.889693:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775768115.889694:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000100:00000001:1.0:1775768115.889695:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000001:2.0:1775768115.889699:0:13471:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:3.0:1775768115.889699:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000001:1.0:1775768115.889700:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768115.889703:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000020:00000001:3.0:1775768115.889704:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 02000000:00000001:1.0:1775768115.889708:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00020000:00000001:2.0:1775768115.889708:0:13471:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00000100:00000040:3.0:1775768115.889708:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768115.889712:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1775768115.889714:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00020000:00200000:2.0:1775768115.889714:0:13471:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00000040:3.0:1775768115.889715:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768115.889721:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 02000000:00000001:1.0:1775768115.889722:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775768115.889722:0:13471:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:2.0:1775768115.889729:0:13471:0:(lov_io.c:155:lov_sub_get()) Process entered 00000100:00100000:1.0:1775768115.889730:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294275840:192.168.202.156@tcp:250:kworker.0 00000020:00000040:3.0:1775768115.889730:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00020000:00000001:2.0:1775768115.889733:0:13471:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000020:00000001:2.0:1775768115.889736:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00080000:3.0:1775768115.889737:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000020:00000010:2.0:1775768115.889742:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994210b32090. 00000100:00000001:1.0:1775768115.889743:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1775768115.889745:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768115.889748:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000020:00000010:2.0:1775768115.889749:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9942060d4c00. 02000000:00000001:1.0:1775768115.889751:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775768115.889754:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768115.889755:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1775768115.889756:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994209d2de00. 02000000:00000001:3.0:1775768115.889758:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768115.889762:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:1.0:1775768115.889764:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000010:2.0:1775768115.889764:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942182d6ea0. 00000008:00100000:3.0:1775768115.889767:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775768115.889770:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:2.0:1775768115.889771:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207cc8b28. 00000100:00000001:3.0:1775768115.889783:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775768115.889786:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543e400. 00000100:00000010:3.0:1775768115.889789:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ab480. 02000000:00000001:1.0:1775768115.889790:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775768115.889794:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:3.0:1775768115.889800:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000200:1.0:1775768115.889802:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:3.0:1775768115.889803:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:2.0:1775768115.889804:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994220156840. 00000100:00000001:3.0:1775768115.889808:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000010:1.0:1775768115.889809:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00008000:00000010:2.0:1775768115.889811:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942077a9530. 02000000:00000001:3.0:1775768115.889812:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775768115.889817:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:2.0:1775768115.889818:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994208747738. 00000400:00000010:1.0:1775768115.889822:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000080:00000010:2.0:1775768115.889825:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077ba910. 00000020:00000010:2.0:1775768115.889829:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994209d2d000. 00000008:00000010:2.0:1775768115.889833:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942077cd8c0. 02000000:00000010:3.0:1775768115.889833:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2c00. 00020000:00000010:2.0:1775768115.889840:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff99420876e400. 00000100:00000001:3.0:1775768115.889844:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:2.0:1775768115.889845:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076c3f0. 00000100:00000200:1.0:1775768115.889850:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294275840, portal 25 00000080:00000010:2.0:1775768115.889851:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942050663c0. 00000100:00100000:3.0:1775768115.889851:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ab480 x1862022294275968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775768115.889857:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000020:00000001:2.0:1775768115.889858:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107759906968 : -112965949644648 : ffff994210b32098) 00000100:00000001:1.0:1775768115.889862:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640856064 : -112966068695552 : ffff9942099a8e00) 00000020:00001000:2.0:1775768115.889864:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff994210b32090 00000100:00000001:3.0:1775768115.889870:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000020:00000001:2.0:1775768115.889871:0:13471:0:(cl_object.c:151:cl_object_top()) ffff99420791ae90 -> ffff99420791a5f8 00000100:00000001:3.0:1775768115.889875:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768115.889876:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000100:00000040:1.0:1775768115.889878:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a8e00 x1862022294275840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768140 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1775768115.889881:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000100:00000040:3.0:1775768115.889882:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000067a0ba74] to pc [ptlrpcd_rcv+-1] req@ffff9942099ab480 x1862022294275968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1775768115.889885:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:2.0:1775768115.889889:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775768115.889895:0:13471:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775768115.889901:0:13471:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107621738216 : -112966087813400 : ffff99420876d6e8) 00000020:00000001:2.0:1775768115.889908:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000020:00000001:2.0:1775768115.889912:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768115.889913:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768115.889914:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00020000:00000001:2.0:1775768115.889918:0:13471:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768115.889918:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000001:2.0:1775768115.889924:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768115.889926:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000020:00000001:2.0:1775768115.889929:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00000100:00000040:1.0:1775768115.889930:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000100:00000001:3.0:1775768115.889931:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00020000:00000001:2.0:1775768115.889933:0:13471:0:(lov_io.c:1108:lov_io_lock()) Process entered 00000100:00080000:3.0:1775768115.889935:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000010:1.0:1775768115.889936:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00020000:00000001:2.0:1775768115.889937:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775768115.889939:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00000100:00000200:1.0:1775768115.889943:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294275840, offset 0 00000020:00000001:2.0:1775768115.889943:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:2.0:1775768115.889945:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768115.889949:0:36:0:(import.c:484:import_select_connection()) Process entered 00000020:00000001:2.0:1775768115.889950:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775768115.889955:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768115.889956:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00080000:3.0:1775768115.889957:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5368 00020000:00000001:2.0:1775768115.889960:0:13471:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768115.889964:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:2.0:1775768115.889966:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775768115.889968:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:2.0:1775768115.889970:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768115.889975:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:2.0:1775768115.889976:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00000400:00000200:3.0:1775768115.889976:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00020000:00000001:2.0:1775768115.889981:0:13471:0:(lov_io.c:1114:lov_io_start()) Process entered 00000100:00080000:3.0:1775768115.889982:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00020000:00000001:2.0:1775768115.889984:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775768115.889987:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00000400:00000200:1.0:1775768115.889988:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00000040:3.0:1775768115.889989:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000008:00000001:2.0:1775768115.889991:0:13471:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000100:00000001:3.0:1775768115.889994:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000008:00000001:2.0:1775768115.889996:0:13471:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000100:00000040:3.0:1775768115.889997:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000200:1.0:1775768115.890000:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1775768115.890001:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000008:00000001:2.0:1775768115.890005:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000020:00000001:3.0:1775768115.890006:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775768115.890009:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000008:00000020:2.0:1775768115.890010:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000400:00000200:1.0:1775768115.890011:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000020:00000001:3.0:1775768115.890011:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000008:00000001:2.0:1775768115.890015:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1775768115.890015:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000400:00000200:1.0:1775768115.890021:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000020:00000001:3.0:1775768115.890021:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000008:00000001:2.0:1775768115.890023:0:13471:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000020:00000001:3.0:1775768115.890026:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000008:00000001:2.0:1775768115.890027:0:13471:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000400:00000200:1.0:1775768115.890029:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000008:00000001:2.0:1775768115.890032:0:13471:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107754940544 : -112965954611072 : ffff994210675880) 00000100:00000040:3.0:1775768115.890033:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768115.890039:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1775768115.890040:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225328:225328:256:4294967295] 192.168.202.156@tcp LPNI seq info [225324:225324:8:4294967295] 00000008:00000020:2.0:1775768115.890040:0:13471:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994210675880 ready 0|-|- wr 1|-|+ rd 0|- 10 in flight 00000100:00000040:3.0:1775768115.890043:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768115.890049:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000008:00000001:2.0:1775768115.890050:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:2.0:1775768115.890053:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000020:00000040:3.0:1775768115.890054:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000400:00000200:1.0:1775768115.890055:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00000001:2.0:1775768115.890056:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775768115.890061:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000008:00000001:2.0:1775768115.890062:0:13471:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000100:00000001:3.0:1775768115.890063:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000008:00000020:2.0:1775768115.890067:0:13471:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994218232fc0 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 02000000:00000001:3.0:1775768115.890067:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775768115.890072:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00000001:2.0:1775768115.890077:0:13471:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1775768115.890078:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768115.890081:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000008:00000001:2.0:1775768115.890083:0:13471:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768115.890084:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775768115.890088:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000020:00000001:2.0:1775768115.890090:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1775768115.890091:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775768115.890093:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775768115.890095:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00020000:00000001:2.0:1775768115.890096:0:13471:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768115.890100:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768115.890101:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768115.890104:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00000800:00000010:1.0:1775768115.890105:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000100:00000010:3.0:1775768115.890106:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a9180. 00020000:00000001:2.0:1775768115.890108:0:13471:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00020000:00000001:2.0:1775768115.890111:0:13471:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000020:00000040:3.0:1775768115.890111:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:2.0:1775768115.890115:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 02000000:00000001:3.0:1775768115.890116:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000800:00000200:1.0:1775768115.890118:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000001:2.0:1775768115.890119:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 02000000:00000001:3.0:1775768115.890119:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775768115.890123:0:13471:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768115.890124:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000800:00000200:1.0:1775768115.890126:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00020000:00000001:2.0:1775768115.890127:0:13471:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 02000000:00000001:3.0:1775768115.890128:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000020:00000001:2.0:1775768115.890132:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 02000000:00000001:3.0:1775768115.890132:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768115.890135:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 02000000:00000010:3.0:1775768115.890137:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2800. 00000800:00000200:1.1:1775768115.890138:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00020000:00000001:2.0:1775768115.890139:0:13471:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00000100:00000001:3.0:1775768115.890141:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775768115.890142:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775768115.890146:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000800:00000200:1.1:1775768115.890146:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000020:00000001:2.0:1775768115.890150:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000100:00100000:3.0:1775768115.890150:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a9180 x1862022294276096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00020000:00000001:2.0:1775768115.890179:0:13471:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1775768115.890182:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00020000:00000001:2.0:1775768115.890184:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1775768115.890189:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00020000:00000001:2.0:1775768115.890190:0:13471:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000020:00000001:2.0:1775768115.890193:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000100:00000001:3.0:1775768115.890196:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000020:00000001:2.0:1775768115.890197:0:13471:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000400:00000200:1.0:1775768115.890200:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000001:2.0:1775768115.890201:0:13471:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000100:00000001:3.0:1775768115.890201:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768115.890205:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:2.0:1775768115.890208:0:13471:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00000100:00000040:3.0:1775768115.890210:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bdc2586] to pc [ptlrpcd_rcv+-1] req@ffff9942099a9180 x1862022294276096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:2.0:1775768115.890211:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000400:00000200:1.0:1775768115.890213:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000020:00000001:2.0:1775768115.890214:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000001:2.0:1775768115.890217:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00020000:00000001:2.0:1775768115.890221:0:13471:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768115.890225:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00020000:00000001:2.0:1775768115.890225:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768115.890227:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775768115.890229:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00020000:00000001:2.0:1775768115.890230:0:13471:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000100:00080000:3.0:1775768115.890232:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000001:2.0:1775768115.890235:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000100:00000001:1.0:1775768115.890238:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:3.0:1775768115.890238:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000020:00000001:2.0:1775768115.890239:0:13471:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768115.890243:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000020:00000001:2.0:1775768115.890244:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00000100:00080000:3.0:1775768115.890246:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:1.0:1775768115.890248:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a8e00 x1862022294275840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768140 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:2.0:1775768115.890248:0:13471:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:2.0:1775768115.890251:0:13471:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000100:00000001:3.0:1775768115.890253:0:36:0:(import.c:484:import_select_connection()) Process entered 00000020:00000001:2.0:1775768115.890255:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:2.0:1775768115.890259:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000100:00080000:3.0:1775768115.890259:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5368 00000020:00001000:2.0:1775768115.890264:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff994210b32090 00000400:00000200:3.0:1775768115.890269:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000010:2.0:1775768115.890271:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994209d2de00. 00000400:00000200:3.0:1775768115.890275:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:1.0:1775768115.890276:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000020:00000010:2.0:1775768115.890277:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942182d6ea0. 00000100:00080000:3.0:1775768115.890277:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775768115.890284:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a8e00 x1862022294275840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768140 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775768115.890284:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000008:00000010:2.0:1775768115.890285:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207cc8b28. 00000100:00000001:3.0:1775768115.890289:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00020000:00000010:2.0:1775768115.890293:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994220156840. 00000100:00000040:3.0:1775768115.890293:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768115.890299:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00008000:00000010:2.0:1775768115.890301:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942077a9530. 00000020:00000001:3.0:1775768115.890303:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775768115.890306:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:3.0:1775768115.890308:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:1.0:1775768115.890309:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:2.0:1775768115.890309:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994208747738. 00000020:00000040:3.0:1775768115.890310:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000100:00000001:1.0:1775768115.890315:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:3.0:1775768115.890315:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000080:00000010:2.0:1775768115.890317:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077ba910. 00000020:00000001:3.0:1775768115.890320:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000010:2.0:1775768115.890323:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9942060d4c00. 00000100:00000040:3.0:1775768115.890324:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768115.890325:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768115.890328:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775768115.890331:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000008:00000010:2.0:1775768115.890332:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942077cd8c0. 00000400:00000200:1.0:1775768115.890336:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775768115.890336:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00020000:00000010:2.0:1775768115.890338:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff99420876e400. 00000020:00000040:3.0:1775768115.890340:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00000001:1.0:1775768115.890343:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4300042046 00008000:00000010:2.0:1775768115.890344:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076c3f0. 00000100:00080000:3.0:1775768115.890345:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775768115.890350:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:2.0:1775768115.890352:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942050663c0. 02000000:00000001:3.0:1775768115.890352:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000010:1.0:1775768115.890353:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba000. 02000000:00000001:3.0:1775768115.890357:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1775768115.890358:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994209d2d000. 02000000:00000001:3.0:1775768115.890362:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1775768115.890363:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994210b32090. 00000100:00000001:1.0:1775768115.890364:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1775768115.890365:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00020000:00000001:2.0:1775768115.890369:0:13471:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00000008:00100000:3.0:1775768115.890369:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:3.0:1775768115.890372:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768115.890374:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4300042046 00020000:00000001:2.0:1775768115.890374:0:13471:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000100:00000001:3.0:1775768115.890375:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775768115.890378:0:13471:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000100:00000010:3.0:1775768115.890380:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8700. 00000080:00200000:2.0:1775768115.890382:0:13471:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x10176:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00000010:1.0:1775768115.890383:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba060. 00000020:00000040:3.0:1775768115.890383:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:3.0:1775768115.890390:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1775768115.890391:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000080:00000001:2.0:1775768115.890391:0:13471:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 02000000:00000001:3.0:1775768115.890391:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768115.890394:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000100:00000001:3.0:1775768115.890397:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1775768115.890399:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775768115.890399:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000020:00001000:2.0:1775768115.890400:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff994210b32630 02000000:00000001:3.0:1775768115.890403:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1775768115.890405:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff9942060d5a00. 00000100:00000001:1.0:1775768115.890406:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775768115.890408:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0400. 00000020:00000010:2.0:1775768115.890409:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942182d6a20. 00000100:00000001:1.0:1775768115.890411:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000008:00000010:2.0:1775768115.890416:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207ccc2f0. 00000100:00000001:3.0:1775768115.890417:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:2.0:1775768115.890420:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994220156e40. 00000100:00100000:1.0:1775768115.890421:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768115/real 1775768115] req@ffff9942099a8e00 x1862022294275840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00008000:00000010:2.0:1775768115.890425:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942077a8e20. 00000100:00100000:3.0:1775768115.890428:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8700 x1862022294276224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000080:00000010:2.0:1775768115.890431:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff9942087472a0. 00000080:00000010:2.0:1775768115.890436:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b8000. 00000020:00000010:2.0:1775768115.890442:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9942060d4800. 00000008:00000010:2.0:1775768115.890448:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942077cd600. 00020000:00000010:2.0:1775768115.890454:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff99420876d680. 00000400:00000001:1.0:1775768115.890457:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00008000:00000010:2.0:1775768115.890460:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076cf60. 00000100:00000001:3.0:1775768115.890460:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000001:1.0:1775768115.890461:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1775768115.890465:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:2.0:1775768115.890466:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff994205066870. 00000400:00000010:1.0:1775768115.890467:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000100:00000040:3.0:1775768115.890471:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000faa25e3b] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8700 x1862022294276224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1775768115.890474:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9942060d5200. 00000400:00000200:1.0:1775768115.890475:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768115.890481:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000020:00000010:2.0:1775768115.890481:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994210b32630. 00000100:00000001:3.0:1775768115.890484:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775768115.890488:0:13471:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775768115.890488:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775768115.890489:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000080:00000001:2.0:1775768115.890495:0:13471:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768115.890495:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000200:1.0:1775768115.890499:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a8e00 x1862022294275840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768115.890499:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775768115.890501:0:36:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775768115.890506:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775768115.890511:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5368 00000400:00000200:3.0:1775768115.890519:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775768115.890526:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000200:1.0:1775768115.890528:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a8e00 x1862022294275840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768115.890528:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775768115.890533:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768115.890538:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775768115.890542:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768115.890545:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775768115.890548:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:1.0:1775768115.890549:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000040:3.0:1775768115.890549:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000100:00000001:1.0:1775768115.890553:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1775768115.890553:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775768115.890554:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:3.0:1775768115.890558:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000001:1.0:1775768115.890560:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:3.0:1775768115.890560:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000001:1.0:1775768115.890564:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775768115.890564:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768115.890568:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775768115.890572:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00100000:1.0:1775768115.890575:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a8e00 x1862022294275840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768115.890575:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775768115.890578:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:3.0:1775768115.890583:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775768115.890587:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768115.890590:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775768115.890594:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768115.890596:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1775768115.890597:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768115.890600:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775768115.890602:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:3.0:1775768115.890605:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768115.890606:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a8e00 x1862022294275840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768140 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768115.890606:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775768115.890611:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8380. 00000020:00000040:3.0:1775768115.890615:0:36:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:3.0:1775768115.890619:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775768115.890622:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768115.890624:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775768115.890624:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775768115.890626:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775768115.890628:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768115.890630:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:3.0:1775768115.890632:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b3400. 00000100:00000001:1.0:1775768115.890633:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768115.890637:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768115.890637:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768115.890644:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00100000:3.0:1775768115.890645:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8380 x1862022294276352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775768115.890649:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768115.890656:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1775768115.890657:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775768115.890661:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768115.890665:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055f5621a] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8380 x1862022294276352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768115.890668:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768115.890675:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1775768115.890676:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768115.890682:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5378) 00000100:00000040:1.0:1775768115.890686:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a8e00 x1862022294275840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768140 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768115.890698:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294275840:192.168.202.156@tcp:250:kworker.0 00000100:00000001:1.0:1775768115.890704:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768115.890707:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768115.890710:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a8e00 x1862022294275840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768140 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768115.890717:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768115.890720:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768115.890724:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543e400. 02000000:00000001:1.0:1775768115.890727:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768115.890729:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768115.890731:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:1.0:1775768115.890734:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768115.890736:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b0000. 02000000:00000001:1.0:1775768115.890741:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768115.890743:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768115.890745:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a8e00. 00000100:00000001:1.0:1775768115.890748:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768115.890750:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768115.890752:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768115.890758:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768115.890760:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768115.890763:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768115.890764:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768115.890766:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768115.890770:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ab480 x1862022294275968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768115.890799:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768115.890800:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768115.890803:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768115.890805:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768115.890807:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768115.890811:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ab480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294275968:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768115.890815:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768115.890817:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768115.890819:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768115.890821:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768115.890824:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543e400. 02000000:00000001:1.0:1775768115.890826:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768115.890830:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768115.890834:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000400:00000010:1.0:1775768115.890838:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000200:1.0:1775768115.890841:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294275968, portal 10 00000100:00000001:1.0:1775768115.890843:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768115.890844:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640865920 : -112966068685696 : ffff9942099ab480) 00000100:00000040:1.0:1775768115.890852:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ab480 x1862022294275968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768140 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768115.890859:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768115.890862:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768115.890865:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768115.890867:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294275968, offset 0 00000400:00000200:1.0:1775768115.890871:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768115.890876:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768115.890880:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768115.890884:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768115.890886:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768115.890889:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768115.890892:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768115.890896:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225329:225329:256:4294967295] 192.168.202.156@tcp LPNI seq info [225325:225325:8:4294967295] 00000400:00000200:1.0:1775768115.890901:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768115.890908:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768115.890913:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768115.890917:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000800:00000200:1.0:1775768115.890922:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768115.890925:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775768115.890930:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768115.890933:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1775768115.890937:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768115.890939:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768115.890942:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768115.890946:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768115.890950:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768115.890952:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768115.890954:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768115.890959:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ab480 x1862022294275968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768140 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768115.890967:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768115.890970:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ab480 x1862022294275968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768140 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768115.890977:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768115.890979:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768115.890982:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768115.890986:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:1.0:1775768115.890990:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768115.890992:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300040723 00000100:00000001:1.0:1775768115.890995:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300042046 00000100:00000001:1.0:1775768115.890997:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300040723 00000100:00000001:1.0:1775768115.891000:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300042046 00000100:00000001:1.0:1775768115.891002:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768115.891005:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768115.891007:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768115.891011:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768115/real 1775768115] req@ffff9942099ab480 x1862022294275968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768115.891019:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768115.891021:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768115.891023:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768115.891025:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768115.891027:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768115.891029:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768115.891032:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ab480 x1862022294275968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768115.891042:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ab480 x1862022294275968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768115.891049:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768115.891050:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768115.891052:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768115.891054:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768115.891057:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ab480 x1862022294275968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768115.891065:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768115.891069:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ab480 x1862022294275968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768140 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768115.891077:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768115.891080:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768115.891081:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768115.891083:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768115.891085:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768115.891088:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768115.891091:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768115.891095:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768115.891097:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768115.891102:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ab480 x1862022294275968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768140 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768115.891110:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ab480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294275968:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768115.891115:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768115.891118:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a9180 x1862022294276096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768115.891125:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768115.891127:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768115.891129:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768115.891131:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768115.891133:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768115.891136:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294276096:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768115.891140:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768115.891142:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768115.891143:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768115.891145:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768115.891148:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543d400. 02000000:00000001:1.0:1775768115.891151:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768115.891172:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768115.891176:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000400:00000010:1.0:1775768115.891180:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000200:1.0:1775768115.891183:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294276096, portal 10 00000100:00000001:1.0:1775768115.891185:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768115.891186:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640856960 : -112966068694656 : ffff9942099a9180) 00000100:00000040:1.0:1775768115.891193:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a9180 x1862022294276096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768140 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768115.891200:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768115.891203:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768115.891206:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768115.891209:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294276096, offset 0 00000400:00000200:1.0:1775768115.891212:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768115.891217:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768115.891220:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768115.891224:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768115.891227:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768115.891230:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768115.891232:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768115.891236:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225330:225330:256:4294967295] 192.168.202.156@tcp LPNI seq info [225326:225326:8:4294967295] 00000400:00000200:1.0:1775768115.891241:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768115.891247:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768115.891252:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768115.891255:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000800:00000200:1.0:1775768115.891260:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768115.891263:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775768115.891267:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768115.891270:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1775768115.891274:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768115.891276:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768115.891279:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768115.891283:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768115.891286:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768115.891288:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768115.891290:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768115.891294:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a9180 x1862022294276096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768140 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768115.891302:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768115.891306:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a9180 x1862022294276096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768140 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768115.891313:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768115.891315:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768115.891319:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768115.891322:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:1.0:1775768115.891326:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768115.891328:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300040716 00000100:00000001:1.0:1775768115.891331:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300042046 00000100:00000001:1.0:1775768115.891333:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300040716 00000100:00000001:1.0:1775768115.891335:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300042046 00000100:00000001:1.0:1775768115.891337:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768115.891340:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768115.891342:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768115.891345:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768115/real 1775768115] req@ffff9942099a9180 x1862022294276096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768115.891354:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768115.891355:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768115.891357:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768115.891360:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768115.891361:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768115.891364:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768115.891367:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a9180 x1862022294276096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768115.891376:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a9180 x1862022294276096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768115.891383:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768115.891385:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768115.891387:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768115.891388:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768115.891392:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a9180 x1862022294276096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768115.891400:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768115.891405:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a9180 x1862022294276096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768140 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768115.891412:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768115.891414:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768115.891416:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768115.891418:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768115.891420:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768115.891423:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768115.891426:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768115.891430:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768115.891432:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768115.891436:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a9180 x1862022294276096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768140 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768115.891445:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294276096:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768115.891449:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768115.891453:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a8700 x1862022294276224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768115.891459:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768115.891461:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768115.891464:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768115.891465:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768115.891468:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768115.891471:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294276224:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768115.891474:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768115.891476:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768115.891478:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768115.891480:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768115.891483:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543d000. 02000000:00000001:1.0:1775768115.891486:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768115.891490:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768115.891492:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000400:00000010:1.0:1775768115.891496:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000200:1.0:1775768115.891499:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294276224, portal 4 00000100:00000001:1.0:1775768115.891501:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768115.891502:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640854272 : -112966068697344 : ffff9942099a8700) 00000100:00000040:1.0:1775768115.891508:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a8700 x1862022294276224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768140 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768115.891516:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768115.891518:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768115.891521:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768115.891523:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294276224, offset 0 00000400:00000200:1.0:1775768115.891528:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768115.891532:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768115.891536:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768115.891539:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768115.891542:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768115.891545:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768115.891547:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768115.891550:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225331:225331:256:4294967295] 192.168.202.156@tcp LPNI seq info [225327:225327:8:4294967295] 00000400:00000200:1.0:1775768115.891555:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768115.891561:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768115.891567:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768115.891570:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000800:00000200:1.0:1775768115.891574:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768115.891577:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775768115.891582:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768115.891584:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1775768115.891588:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768115.891590:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768115.891593:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768115.891597:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768115.891600:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768115.891603:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768115.891605:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768115.891609:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a8700 x1862022294276224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768140 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768115.891618:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768115.891621:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a8700 x1862022294276224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768140 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768115.891629:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768115.891631:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768115.891634:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768115.891637:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:1.0:1775768115.891641:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768115.891643:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300040739 00000100:00000001:1.0:1775768115.891645:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300042046 00000100:00000001:1.0:1775768115.891648:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300040739 00000100:00000001:1.0:1775768115.891650:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300042046 00000100:00000001:1.0:1775768115.891653:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768115.891655:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768115.891657:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768115.891661:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768115/real 1775768115] req@ffff9942099a8700 x1862022294276224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768115.891669:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768115.891671:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768115.891673:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768115.891675:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768115.891677:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768115.891679:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768115.891682:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a8700 x1862022294276224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768115.891692:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a8700 x1862022294276224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768115.891698:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768115.891700:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768115.891702:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768115.891704:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768115.891707:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a8700 x1862022294276224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768115.891715:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768115.891719:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a8700 x1862022294276224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768140 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768115.891726:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768115.891729:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768115.891730:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768115.891732:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768115.891734:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768115.891738:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768115.891741:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768115.891744:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768115.891746:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768115.891750:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a8700 x1862022294276224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768140 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768115.891759:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294276224:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768115.891763:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768115.891766:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a8380 x1862022294276352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768115.891788:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768115.891790:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768115.891793:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768115.891794:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768115.891796:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768115.891800:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294276352:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768115.891803:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768115.891805:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768115.891807:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768115.891809:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768115.891813:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543f000. 02000000:00000001:1.0:1775768115.891815:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768115.891819:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768115.891822:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000400:00000010:1.0:1775768115.891826:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000200:1.0:1775768115.891829:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294276352, portal 4 00000100:00000001:1.0:1775768115.891831:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768115.891833:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640853376 : -112966068698240 : ffff9942099a8380) 00000100:00000040:1.0:1775768115.891839:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a8380 x1862022294276352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768140 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768115.891846:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768115.891849:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768115.891852:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768115.891854:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294276352, offset 0 00000400:00000200:1.0:1775768115.891859:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768115.891864:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768115.891868:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768115.891870:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768115.891873:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768115.891875:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768115.891877:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768115.891880:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225332:225332:256:4294967295] 192.168.202.156@tcp LPNI seq info [225328:225328:8:4294967295] 00000400:00000200:1.0:1775768115.891885:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768115.891892:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768115.891897:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768115.891901:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000800:00000200:1.0:1775768115.891905:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768115.891908:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775768115.891912:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768115.891915:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1775768115.891919:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768115.891921:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768115.891924:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768115.891928:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768115.891931:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768115.891933:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768115.891936:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768115.891940:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a8380 x1862022294276352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768140 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768115.891948:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768115.891951:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a8380 x1862022294276352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768140 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768115.891957:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768115.891959:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768115.891962:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768115.891965:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:1.0:1775768115.891969:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768115.891971:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300040742 00000100:00000001:1.0:1775768115.891973:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300042046 00000100:00000001:1.0:1775768115.891976:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300040742 00000100:00000001:1.0:1775768115.891978:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300042046 00000100:00000001:1.0:1775768115.891980:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768115.891982:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768115.891985:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768115.891989:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768115/real 1775768115] req@ffff9942099a8380 x1862022294276352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768115.891998:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768115.891999:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768115.892001:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768115.892004:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768115.892006:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768115.892008:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768115.892012:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a8380 x1862022294276352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768115.892021:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a8380 x1862022294276352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768115.892027:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768115.892029:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768115.892031:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768115.892032:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768115.892036:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a8380 x1862022294276352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768115.892043:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768115.892048:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a8380 x1862022294276352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768140 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768115.892055:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768115.892058:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768115.892060:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768115.892062:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768115.892064:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768115.892068:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768115.892071:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768115.892074:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768115.892077:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768115.892082:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a8380 x1862022294276352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768140 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768115.892091:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294276352:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768115.892095:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768115.892097:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768115.892100:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ab480 x1862022294275968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768140 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768115.892107:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768115.892109:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768115.892111:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543e400. 02000000:00000001:1.0:1775768115.892114:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768115.892116:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768115.892117:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:1.0:1775768115.892120:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768115.892121:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b2c00. 02000000:00000001:1.0:1775768115.892124:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768115.892126:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768115.892127:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ab480. 00000100:00000001:1.0:1775768115.892130:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768115.892132:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768115.892134:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768115.892137:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a9180 x1862022294276096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768140 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768115.892144:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768115.892146:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768115.892148:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543d400. 02000000:00000001:1.0:1775768115.892150:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768115.892152:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768115.892153:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:1.0:1775768115.892174:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768115.892176:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b2800. 02000000:00000001:1.0:1775768115.892180:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768115.892182:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768115.892183:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a9180. 00000100:00000001:1.0:1775768115.892186:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768115.892187:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768115.892189:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768115.892193:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a8700 x1862022294276224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768140 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768115.892201:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768115.892202:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768115.892205:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543d000. 02000000:00000001:1.0:1775768115.892207:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768115.892208:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768115.892210:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:1.0:1775768115.892212:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768115.892213:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b0400. 02000000:00000001:1.0:1775768115.892216:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768115.892217:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768115.892219:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a8700. 00000100:00000001:1.0:1775768115.892221:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768115.892223:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768115.892224:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768115.892227:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a8380 x1862022294276352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768140 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768115.892234:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768115.892235:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768115.892237:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543f000. 02000000:00000001:1.0:1775768115.892239:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768115.892241:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768115.892242:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:1.0:1775768115.892244:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768115.892245:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b3400. 02000000:00000001:1.0:1775768115.892248:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768115.892249:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768115.892251:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a8380. 00000100:00000001:1.0:1775768115.892253:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768115.892254:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768115.892257:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768115.892261:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768115.892263:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768115.892265:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768115.892267:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768115.953301:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768115.953307:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768115.953314:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768115.953315:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768115.953320:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768115.953321:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768115.953326:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768115.953328:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768115.953331:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768115.953332:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768115.953406:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768115.953410:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768115.953414:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768116.017265:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768116.017271:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768116.017272:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768116.017276:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768116.017279:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768116.017282:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768116.017284:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768116.017287:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768116.017289:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768116.913281:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768116.913281:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768116.913282:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768116.913291:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768116.913291:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768116.913291:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768116.913294:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768116.913295:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768116.913295:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768116.913304:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:0.0:1775768116.913305:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:2.0:1775768116.913305:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768116.913312:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768116.913312:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768116.913312:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768116.913319:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768116.913319:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768116.913323:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768116.913323:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768116.913327:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768116.913331:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768116.913331:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768116.913334:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768116.913337:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768116.913338:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768116.913341:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768116.913345:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768116.977269:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768116.977276:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768116.977281:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768116.977284:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768116.977287:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768116.977290:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768116.977293:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768116.977295:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768116.977297:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768116.977299:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768116.977302:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768116.977303:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768116.977304:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768116.977305:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768116.977306:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768116.977307:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768116.977309:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768116.977310:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768116.977313:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768116.977315:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768116.977316:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768116.977318:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768116.977320:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768116.977321:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768116.977323:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768116.977325:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768116.977329:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768116.977330:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768116.977332:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768116.977334:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768116.977335:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768116.977337:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768116.977339:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768116.977341:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768116.977411:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768116.977416:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768116.977420:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768117.041315:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768117.041323:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768117.041325:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768117.041332:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768117.041337:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768117.041342:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768117.041345:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768117.041348:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768117.041352:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768117.233325:0:14953:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4300042046, at 4300042048 00000100:00000040:1.0:1775768117.233325:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4300042046, at 4300042048 00000100:00000010:1.0:1775768117.233345:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cba060. 00000100:00000010:0.0:1775768117.233348:0:14953:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cba000. 00000100:00000001:0.0:1775768117.937283:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768117.937283:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768117.937283:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768117.937291:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768117.937291:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768117.937291:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768117.937295:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768117.937295:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768117.937296:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768117.937304:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:0.0:1775768117.937305:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:2.0:1775768117.937305:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768117.937311:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768117.937311:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768117.937312:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768117.937315:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768117.937316:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768117.937316:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768117.937321:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768117.937324:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768117.937326:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768117.937328:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768117.937331:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768117.937334:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768117.937334:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768117.937338:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768117.937341:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768118.001255:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768118.001261:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768118.001266:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768118.001267:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768118.001270:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768118.001271:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768118.001273:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768118.001274:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768118.001277:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768118.001277:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768118.001339:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768118.001343:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768118.001347:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768118.065232:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768118.065238:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768118.065240:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768118.065245:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768118.065248:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768118.065252:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768118.065254:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768118.065256:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768118.065258:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768118.961254:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768118.961257:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768118.961258:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768118.961263:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768118.961263:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768118.961264:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768118.961266:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768118.961267:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768118.961267:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768118.961274:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:0.0:1775768118.961275:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:1.0:1775768118.961275:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768118.961280:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768118.961281:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768118.961281:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768118.961284:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768118.961285:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768118.961286:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768118.961289:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768118.961293:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768118.961295:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768118.961299:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768118.961302:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768118.961305:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768118.961306:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768118.961310:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768118.961313:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768119.025292:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768119.025298:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768119.025303:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768119.025304:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768119.025307:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768119.025308:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768119.025312:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768119.025313:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768119.025316:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768119.025317:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768119.025369:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768119.025373:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768119.025376:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768119.089290:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768119.089297:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768119.089299:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768119.089304:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768119.089308:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768119.089312:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768119.089315:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768119.089317:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768119.089319:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768119.985248:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768119.985248:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768119.985250:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768119.985254:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768119.985254:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768119.985255:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768119.985256:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768119.985258:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768119.985259:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768119.985261:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:0.0:1775768119.985263:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768119.985264:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:2.0:1775768119.985264:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768119.985268:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768119.985269:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768119.985269:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768119.985271:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768119.985273:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768119.985274:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768119.985275:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768119.985279:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768119.985280:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768119.985283:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768119.985286:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768119.985287:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768119.985290:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768119.985293:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768120.049253:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768120.049258:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768120.049262:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768120.049263:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768120.049265:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768120.049266:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768120.049269:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768120.049270:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768120.049273:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768120.049273:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768120.049316:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768120.049318:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768120.049321:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768120.113246:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768120.113251:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768120.113253:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768120.113258:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768120.113261:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768120.113265:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768120.113268:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768120.113270:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768120.113272:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768121.009218:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768121.009223:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768121.009225:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768121.009230:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775768121.009230:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:3.0:1775768121.009232:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775768121.009234:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768121.009235:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768121.009238:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768121.009239:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775768121.009240:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768121.009242:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:2.0:1775768121.009242:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775768121.009243:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1775768121.009244:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768121.009246:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768121.009247:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768121.009248:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768121.009248:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1775768121.009248:0:36:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768121.009252:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768121.009252:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768121.009253:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768121.009254:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775768121.009258:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768121.009259:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768121.009262:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775768121.009262:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5373 00000100:00000001:1.0:1775768121.009263:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768121.009268:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768121.009269:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768121.009271:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1775768121.009274:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768121.009275:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775768121.009281:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775768121.009284:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775768121.009286:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768121.009289:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000080:00000001:0.0:1775768121.009292:0:13471:0:(rw.c:1481:ll_writepages()) Process entered 00000100:00000040:3.0:1775768121.009292:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768121.009295:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000080:00000001:0.0:1775768121.009296:0:13471:0:(file.c:5250:cl_sync_file_range()) Process entered 00000020:00000001:3.0:1775768121.009298:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000001:0.0:1775768121.009300:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000040:3.0:1775768121.009300:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:3.0:1775768121.009303:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775768121.009305:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:3.0:1775768121.009306:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:3.0:1775768121.009308:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000010:0.0:1775768121.009310:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff9942103bed80. 00000100:00000040:3.0:1775768121.009310:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768121.009313:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775768121.009315:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768121.009316:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000010:0.0:1775768121.009318:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0f600. 00000020:00000040:3.0:1775768121.009318:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:3.0:1775768121.009322:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775768121.009324:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1775768121.009326:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210b0fc00. 02000000:00000001:3.0:1775768121.009326:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775768121.009329:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1775768121.009332:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942078a3d80. 02000000:00000001:3.0:1775768121.009332:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768121.009335:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775768121.009337:0:36:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775768121.009341:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8380. 00000020:00000040:3.0:1775768121.009344:0:36:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:3.0:1775768121.009346:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775768121.009347:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:0.0:1775768121.009349:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff9942086e6468. 00000100:00000001:3.0:1775768121.009349:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775768121.009353:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00020000:00000010:0.0:1775768121.009354:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994209592f00. 02000000:00000001:3.0:1775768121.009357:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:0.0:1775768121.009359:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff99422035b880. 02000000:00000010:3.0:1775768121.009363:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b1000. 00000080:00000010:0.0:1775768121.009365:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994207e3e150. 00000080:00000010:0.0:1775768121.009369:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b2238. 00000100:00000001:3.0:1775768121.009369:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1775768121.009375:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0ec00. 00000008:00000010:0.0:1775768121.009378:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff99420906be40. 00000100:00100000:3.0:1775768121.009379:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8380 x1862022294276480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00020000:00000010:0.0:1775768121.009382:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff9942103ccfc0. 00008000:00000010:0.0:1775768121.009387:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076afa0. 00000100:00000001:3.0:1775768121.009390:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000080:00000010:0.0:1775768121.009391:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942107690f0. 00000020:00000001:0.0:1775768121.009394:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107752095112 : -112965957456504 : ffff9942103bed88) 00000100:00000001:3.0:1775768121.009396:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1775768121.009397:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff9942103bed80 00000020:00000001:0.0:1775768121.009402:0:13471:0:(cl_object.c:151:cl_object_top()) ffff9942078a76c0 -> ffff9942078a76c0 00000100:00000040:3.0:1775768121.009404:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055f5621a] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8380 x1862022294276480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775768121.009405:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1775768121.009407:0:13471:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:0.0:1775768121.009409:0:13471:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1775768121.009415:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1775768121.009417:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1775768121.009419:0:13471:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1775768121.009424:0:13471:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017a:0x0]io ffff9942077b2298 type 6 ignore/verify layout 1/0 00020000:00000002:0.0:1775768121.009427:0:13471:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207c1f9b0) owner 0000000000000000/ffff994220080000 00020000:00000001:0.0:1775768121.009429:0:13471:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1775768121.009430:0:13471:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1775768121.009432:0:13471:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00020000:00000001:0.0:1775768121.009433:0:13471:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768121.009434:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768121.009435:0:13471:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1775768121.009439:0:13471:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1775768121.009440:0:13471:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768121.009440:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000001:0.0:1775768121.009443:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1775768121.009445:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1775768121.009447:0:13471:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00000100:00080000:3.0:1775768121.009447:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00020000:00000002:0.0:1775768121.009450:0:13471:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207c1f9b0) owner 0000000000000000/ffff994220080000 00000100:00000001:3.0:1775768121.009450:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000020:00000001:0.0:1775768121.009453:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768121.009453:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768121.009454:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1775768121.009458:0:13471:0:(cl_io.c:823:cl_io_loop()) Process entered 00000100:00000001:1.0:1775768121.009459:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768121.009459:0:36:0:(import.c:484:import_select_connection()) Process entered 00000020:00000001:0.0:1775768121.009460:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000100:00000001:1.0:1775768121.009463:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00080000:3.0:1775768121.009463:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5373 00020000:00000001:0.0:1775768121.009464:0:13471:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00020000:00200000:0.0:1775768121.009467:0:13471:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000400:00000200:3.0:1775768121.009470:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000002:0.0:1775768121.009472:0:13471:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00000100:00000040:1.0:1775768121.009474:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a8380 x1862022294276480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1775768121.009475:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00020000:00000001:0.0:1775768121.009476:0:13471:0:(lov_io.c:155:lov_sub_get()) Process entered 00020000:00000001:0.0:1775768121.009479:0:13471:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000100:00080000:3.0:1775768121.009480:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000020:00000001:0.0:1775768121.009481:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:0.0:1775768121.009485:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff9942103be000. 00000100:00000040:3.0:1775768121.009486:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000010:0.0:1775768121.009491:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0ea00. 00000100:00000001:3.0:1775768121.009491:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000010:0.0:1775768121.009496:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210b0fa00. 00000100:00000040:3.0:1775768121.009496:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768121.009498:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000010:0.0:1775768121.009501:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942078a30c0. 00000100:00000001:3.0:1775768121.009501:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:1.0:1775768121.009502:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:0.0:1775768121.009506:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff9942086e4e18. 00000020:00000001:3.0:1775768121.009506:0:36:0:(genops.c:811:class_conn2export()) Process entered 02000000:00000001:1.0:1775768121.009507:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:3.0:1775768121.009509:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00020000:00000010:0.0:1775768121.009510:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994209592e40. 02000000:00000001:1.0:1775768121.009511:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1775768121.009511:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775768121.009514:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 02000000:00000001:1.0:1775768121.009515:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:0.0:1775768121.009516:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994220358000. 00000020:00000001:3.0:1775768121.009516:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000080:00000010:0.0:1775768121.009521:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994207e3e3f0. 00000020:00000001:3.0:1775768121.009521:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00100000:1.0:1775768121.009523:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294276480:192.168.202.156@tcp:250:kworker.0 00000080:00000010:0.0:1775768121.009525:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b0ff8. 00000100:00000040:3.0:1775768121.009526:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000010:0.0:1775768121.009530:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0e000. 00000100:00000001:3.0:1775768121.009530:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775768121.009531:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000008:00000010:0.0:1775768121.009535:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff99420906a6e0. 00000100:00000040:3.0:1775768121.009535:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000001:1.0:1775768121.009537:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00020000:00000010:0.0:1775768121.009540:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff9942103cc480. 02000000:00000001:1.0:1775768121.009540:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768121.009540:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00008000:00000010:0.0:1775768121.009545:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076adb0. 00000020:00000040:3.0:1775768121.009545:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 02000000:00000001:1.0:1775768121.009546:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000080:00000010:0.0:1775768121.009550:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff994210769870. 00000100:00080000:3.0:1775768121.009550:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 02000000:00000010:1.0:1775768121.009552:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543d000. 00000020:00000001:0.0:1775768121.009554:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107752091656 : -112965957459960 : ffff9942103be008) 02000000:00000001:1.0:1775768121.009555:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768121.009555:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768121.009559:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000020:00001000:0.0:1775768121.009560:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff9942103be000 00000020:00000001:0.0:1775768121.009562:0:13471:0:(cl_object.c:151:cl_object_top()) ffff9942078a7788 -> ffff9942078a76c0 00000400:00000200:1.0:1775768121.009564:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:0.0:1775768121.009565:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 02000000:00000001:3.0:1775768121.009565:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1775768121.009569:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 02000000:00000001:3.0:1775768121.009569:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775768121.009570:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000100:00000001:3.0:1775768121.009570:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000020:00000001:0.0:1775768121.009572:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:0.0:1775768121.009574:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1775768121.009575:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00020000:00000001:0.0:1775768121.009576:0:13471:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775768121.009577:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000008:00000001:3.0:1775768121.009580:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768121.009581:0:13471:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107752153128 : -112965957398488 : ffff9942103cd028) 00000100:00000001:3.0:1775768121.009584:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768121.009585:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000100:00000200:1.0:1775768121.009585:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294276480, portal 25 00000020:00000001:0.0:1775768121.009588:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768121.009589:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000010:3.0:1775768121.009590:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8700. 00020000:00000001:0.0:1775768121.009591:0:13471:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768121.009591:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640853376 : -112966068698240 : ffff9942099a8380) 00000020:00000040:3.0:1775768121.009594:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:0.0:1775768121.009596:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768121.009598:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 02000000:00000001:3.0:1775768121.009598:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000040:1.0:1775768121.009601:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a8380 x1862022294276480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768146 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:0.0:1775768121.009602:0:13471:0:(lov_io.c:1108:lov_io_lock()) Process entered 02000000:00000001:3.0:1775768121.009602:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768121.009605:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000001:3.0:1775768121.009606:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000020:00000001:0.0:1775768121.009608:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 02000000:00000001:3.0:1775768121.009608:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000020:00000001:0.0:1775768121.009610:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 02000000:00000001:3.0:1775768121.009612:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768121.009613:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775768121.009617:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0c00. 00000020:00000001:0.0:1775768121.009618:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768121.009619:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00020000:00000001:0.0:1775768121.009622:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768121.009623:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768121.009626:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00020000:00000001:0.0:1775768121.009627:0:13471:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768121.009629:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000400:00000010:1.0:1775768121.009630:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000020:00000001:0.0:1775768121.009632:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768121.009632:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8700 x1862022294276608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:1.0:1775768121.009633:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294276480, offset 0 00000020:00000001:0.0:1775768121.009636:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768121.009640:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00020000:00000001:0.0:1775768121.009643:0:13471:0:(lov_io.c:1114:lov_io_start()) Process entered 00000400:00000200:1.0:1775768121.009645:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00000001:3.0:1775768121.009645:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00020000:00000001:0.0:1775768121.009646:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:0.0:1775768121.009648:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00000100:00000001:3.0:1775768121.009649:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1775768121.009650:0:13471:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000008:00000001:0.0:1775768121.009653:0:13471:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000100:00000040:3.0:1775768121.009654:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000faa25e3b] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8700 x1862022294276608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775768121.009655:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00000001:0.0:1775768121.009658:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:0.0:1775768121.009660:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:0.0:1775768121.009663:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1775768121.009663:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000008:00000001:0.0:1775768121.009668:0:13471:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000008:00000001:0.0:1775768121.009670:0:13471:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000400:00000200:1.0:1775768121.009671:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000008:00000001:0.0:1775768121.009672:0:13471:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107884691392 : -112965824860224 : ffff994218232fc0) 00000008:00000020:0.0:1775768121.009676:0:13471:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994218232fc0 ready 0|-|- wr 1|-|+ rd 0|- 10 in flight 00000100:00000001:3.0:1775768121.009676:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768121.009677:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000100:00080000:3.0:1775768121.009681:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000008:00000001:0.0:1775768121.009682:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:0.0:1775768121.009684:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000400:00000200:1.0:1775768121.009685:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000008:00000001:0.0:1775768121.009686:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775768121.009686:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775768121.009689:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000008:00000001:0.0:1775768121.009690:0:13471:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000400:00000200:1.0:1775768121.009692:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00080000:3.0:1775768121.009692:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000008:00000020:0.0:1775768121.009694:0:13471:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994210675880 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000100:00000001:3.0:1775768121.009696:0:36:0:(import.c:484:import_select_connection()) Process entered 00000008:00000001:0.0:1775768121.009700:0:13471:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775768121.009700:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5373 00000008:00000001:0.0:1775768121.009703:0:13471:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768121.009704:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225333:225333:256:4294967295] 192.168.202.156@tcp LPNI seq info [225329:225329:8:4294967295] 00000020:00000001:0.0:1775768121.009708:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775768121.009708:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:0.0:1775768121.009712:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775768121.009714:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00020000:00000001:0.0:1775768121.009715:0:13471:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768121.009719:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768121.009719:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00080000:3.0:1775768121.009720:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000020:00000001:0.0:1775768121.009735:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00000100:00000040:3.0:1775768121.009735:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00020000:00000001:0.0:1775768121.009738:0:13471:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00000100:00000001:3.0:1775768121.009738:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00020000:00000001:0.0:1775768121.009740:0:13471:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000020:00000001:0.0:1775768121.009743:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00000100:00000040:3.0:1775768121.009743:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000001:0.0:1775768121.009746:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00000100:00000001:3.0:1775768121.009747:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00020000:00000001:0.0:1775768121.009748:0:13471:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768121.009751:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000020:00000001:3.0:1775768121.009752:0:36:0:(genops.c:811:class_conn2export()) Process entered 00020000:00000001:0.0:1775768121.009753:0:13471:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000020:00000001:0.0:1775768121.009756:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00000020:00000040:3.0:1775768121.009756:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:0.0:1775768121.009759:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000020:00000001:3.0:1775768121.009759:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000800:00000200:1.0:1775768121.009760:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000020:00000040:3.0:1775768121.009762:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00020000:00000001:0.0:1775768121.009763:0:13471:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00020000:00000001:0.0:1775768121.009765:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000800:00000010:1.0:1775768121.009766:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000020:00000001:3.0:1775768121.009766:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:0.0:1775768121.009767:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000020:00000001:0.0:1775768121.009769:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:0.0:1775768121.009771:0:13471:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768121.009771:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00020000:00000001:0.0:1775768121.009776:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775768121.009777:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00020000:00000001:0.0:1775768121.009778:0:13471:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000100:00000040:3.0:1775768121.009778:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000001:0.0:1775768121.009781:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000100:00000001:3.0:1775768121.009782:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:0.0:1775768121.009783:0:13471:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000800:00000200:1.0:1775768121.009784:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000020:00000001:0.0:1775768121.009786:0:13471:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000100:00000040:3.0:1775768121.009786:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000001:0.0:1775768121.009789:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000100:00000001:3.0:1775768121.009790:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00020000:00000001:0.0:1775768121.009791:0:13471:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00020000:00000001:0.0:1775768121.009794:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:0.0:1775768121.009795:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000040:3.0:1775768121.009795:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000800:00000200:1.1:1775768121.009795:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000001:0.0:1775768121.009798:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000800:00000200:1.1:1775768121.009800:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00020000:00000001:0.0:1775768121.009801:0:13471:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768121.009801:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00020000:00000001:0.0:1775768121.009806:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768121.009807:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1775768121.009808:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00020000:00000001:0.0:1775768121.009810:0:13471:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 02000000:00000001:3.0:1775768121.009811:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000020:00000001:0.0:1775768121.009813:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000800:00000010:1.0:1775768121.009813:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000020:00000001:0.0:1775768121.009815:0:13471:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768121.009817:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 02000000:00000001:3.0:1775768121.009817:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1775768121.009819:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00020000:00000001:0.0:1775768121.009820:0:13471:0:(lov_io.c:675:lov_io_fini()) Process entered 02000000:00000001:3.0:1775768121.009821:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768121.009823:0:13471:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000100:00000001:3.0:1775768121.009823:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000020:00000001:0.0:1775768121.009824:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:0.0:1775768121.009827:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000400:00000200:1.0:1775768121.009827:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000008:00100000:3.0:1775768121.009827:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000020:00001000:0.0:1775768121.009830:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff9942103be000 00000400:00000200:1.0:1775768121.009832:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000020:00000010:0.0:1775768121.009833:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210b0fa00. 00000008:00000001:3.0:1775768121.009834:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775768121.009836:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:3.0:1775768121.009838:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1775768121.009839:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942078a30c0. 00000100:00000001:1.0:1775768121.009841:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000008:00000010:0.0:1775768121.009842:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff9942086e4e18. 00000100:00000010:3.0:1775768121.009842:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a9180. 00000020:00000040:3.0:1775768121.009847:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00020000:00000010:0.0:1775768121.009849:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994209592e40. 00000100:00000200:1.0:1775768121.009851:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a8380 x1862022294276480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768146 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00008000:00000010:0.0:1775768121.009852:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994220358000. 02000000:00000001:3.0:1775768121.009852:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775768121.009854:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1775768121.009857:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994207e3e3f0. 00000100:00000001:3.0:1775768121.009858:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775768121.009861:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000080:00000010:0.0:1775768121.009862:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b0ff8. 02000000:00000001:3.0:1775768121.009863:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1775768121.009868:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0ea00. 02000000:00000010:3.0:1775768121.009868:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2000. 00000008:00000010:0.0:1775768121.009873:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff99420906a6e0. 00000100:00000001:1.0:1775768121.009873:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1775768121.009874:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:0.0:1775768121.009877:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff9942103cc480. 00000100:00000040:1.0:1775768121.009881:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a8380 x1862022294276480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768146 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768121.009881:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a9180 x1862022294276736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00008000:00000010:0.0:1775768121.009882:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076adb0. 00000080:00000010:0.0:1775768121.009886:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff994210769870. 00000020:00000010:0.0:1775768121.009890:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0e000. 00000020:00000010:0.0:1775768121.009894:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff9942103be000. 00020000:00000001:0.0:1775768121.009898:0:13471:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:0.0:1775768121.009901:0:13471:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:0.0:1775768121.009904:0:13471:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000100:00000001:3.0:1775768121.009904:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000080:00200000:0.0:1775768121.009908:0:13471:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00000001:3.0:1775768121.009908:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768121.009910:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768121.009914:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bdc2586] to pc [ptlrpcd_rcv+-1] req@ffff9942099a9180 x1862022294276736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000001:0.0:1775768121.009916:0:13471:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000100:00000001:1.0:1775768121.009916:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:0.0:1775768121.009919:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1775768121.009923:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff9942103bed80 00000100:00000001:3.0:1775768121.009924:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1775768121.009925:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210b0fc00. 00000100:00000001:1.0:1775768121.009925:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768121.009927:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000010:0.0:1775768121.009930:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942078a3d80. 00000008:00000010:0.0:1775768121.009934:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff9942086e6468. 00000400:00000200:1.0:1775768121.009935:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00080000:3.0:1775768121.009935:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00020000:00000010:0.0:1775768121.009937:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994209592f00. 00008000:00000010:0.0:1775768121.009940:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff99422035b880. 00000100:00000001:3.0:1775768121.009940:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775768121.009941:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4300047165 00000100:00080000:3.0:1775768121.009944:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000080:00000010:0.0:1775768121.009945:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994207e3e150. 00000100:00000010:1.0:1775768121.009948:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba060. 00000080:00000010:0.0:1775768121.009949:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b2238. 00000100:00000001:3.0:1775768121.009950:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775768121.009955:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5373 00000020:00000010:0.0:1775768121.009956:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0f600. 00000100:00000001:1.0:1775768121.009956:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000008:00000010:0.0:1775768121.009962:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff99420906be40. 00000400:00000200:3.0:1775768121.009962:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768121.009964:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4300047165 00020000:00000010:0.0:1775768121.009969:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff9942103ccfc0. 00000100:00000010:1.0:1775768121.009969:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba1e0. 00000400:00000200:3.0:1775768121.009971:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:1.0:1775768121.009975:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00008000:00000010:0.0:1775768121.009976:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076afa0. 00000100:00000001:1.0:1775768121.009979:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000010:0.0:1775768121.009981:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942107690f0. 00000100:00080000:3.0:1775768121.009981:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1775768121.009984:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1775768121.009987:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0ec00. 00000100:00000001:1.0:1775768121.009988:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000040:3.0:1775768121.009988:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000010:0.0:1775768121.009992:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff9942103bed80. 00000100:00000001:3.0:1775768121.009992:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:1.0:1775768121.009997:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768121/real 1775768121] req@ffff9942099a8380 x1862022294276480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775768121.009997:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000080:00000001:0.0:1775768121.009998:0:13471:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768121.009999:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000080:00000001:0.0:1775768121.010002:0:13471:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768121.010003:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775768121.010005:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:3.0:1775768121.010008:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000080:00000001:0.0:1775768121.010011:0:13471:0:(rw.c:1481:ll_writepages()) Process entered 00000020:00000040:3.0:1775768121.010011:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000080:00000001:0.0:1775768121.010013:0:13471:0:(file.c:5250:cl_sync_file_range()) Process entered 00000020:00000001:3.0:1775768121.010014:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:0.0:1775768121.010015:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000001:3.0:1775768121.010018:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000010:0.0:1775768121.010019:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff9942103bed80. 00000400:00000001:1.0:1775768121.010019:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768121.010022:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000010:0.0:1775768121.010023:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0ec00. 00000100:00000040:3.0:1775768121.010023:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000010:1.0:1775768121.010026:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000020:00000010:0.0:1775768121.010028:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210b0f600. 00000100:00000001:3.0:1775768121.010028:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1775768121.010031:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000020:00000010:0.0:1775768121.010032:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942078a3d80. 00000100:00000040:3.0:1775768121.010032:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000010:1.0:1775768121.010036:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:3.0:1775768121.010036:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000008:00000010:0.0:1775768121.010037:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff9942086e6468. 00000020:00000040:3.0:1775768121.010041:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00020000:00000010:0.0:1775768121.010042:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994209592f00. 00000100:00000001:1.0:1775768121.010043:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00080000:3.0:1775768121.010046:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00008000:00000010:0.0:1775768121.010047:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff99422035b880. 00000100:00000200:1.0:1775768121.010051:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a8380 x1862022294276480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768121.010051:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1775768121.010052:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994207e3e150. 00000080:00000010:0.0:1775768121.010056:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b2238. 02000000:00000001:3.0:1775768121.010056:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775768121.010060:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1775768121.010061:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0fc00. 02000000:00000001:3.0:1775768121.010065:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:0.0:1775768121.010066:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff99420906be40. 00000100:00000001:3.0:1775768121.010067:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000200:1.0:1775768121.010069:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a8380 x1862022294276480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000010:0.0:1775768121.010071:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff9942103ccfc0. 00000008:00100000:3.0:1775768121.010071:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00008000:00000010:0.0:1775768121.010075:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076afa0. 00000008:00000001:3.0:1775768121.010076:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1775768121.010080:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942107690f0. 00000100:00000001:3.0:1775768121.010081:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768121.010085:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107752095112 : -112965957456504 : ffff9942103bed88) 00000100:00000001:1.0:1775768121.010086:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000010:3.0:1775768121.010087:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ab480. 00000020:00001000:0.0:1775768121.010088:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff9942103bed80 00000100:00000001:1.0:1775768121.010089:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1775768121.010091:0:13471:0:(cl_object.c:151:cl_object_top()) ffff9942078a7f58 -> ffff9942078a7f58 00000020:00000040:3.0:1775768121.010091:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:0.0:1775768121.010095:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 02000000:00000001:3.0:1775768121.010096:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000080:00000001:0.0:1775768121.010097:0:13471:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:0.0:1775768121.010099:0:13471:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017b:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000100:00000001:1.0:1775768121.010101:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:3.0:1775768121.010101:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768121.010102:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1775768121.010104:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00000001:3.0:1775768121.010105:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000080:00000001:0.0:1775768121.010106:0:13471:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768121.010107:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1775768121.010107:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775768121.010111:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1775768121.010112:0:13471:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017b:0x0]io ffff9942077b2298 type 6 ignore/verify layout 1/0 00000100:00100000:1.0:1775768121.010114:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a8380 x1862022294276480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000002:0.0:1775768121.010117:0:13471:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207c1f000) owner 0000000000000000/ffff994220080000 02000000:00000010:3.0:1775768121.010119:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b3400. 00020000:00000001:0.0:1775768121.010121:0:13471:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1775768121.010124:0:13471:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00000100:00000001:3.0:1775768121.010126:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768121.010127:0:13471:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00020000:00000001:0.0:1775768121.010129:0:13471:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768121.010133:0:13471:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768121.010134:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775768121.010137:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ab480 x1862022294276864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00020000:00000001:0.0:1775768121.010138:0:13471:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1775768121.010140:0:13471:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768121.010144:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000100:00000040:1.0:1775768121.010144:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a8380 x1862022294276480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768146 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775768121.010147:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1775768121.010149:0:13471:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00020000:00000002:0.0:1775768121.010151:0:13471:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207c1f000) owner 0000000000000000/ffff994220080000 00000100:00000001:1.0:1775768121.010172:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768121.010174:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1775768121.010176:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768121.010178:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:0.0:1775768121.010181:0:13471:0:(cl_io.c:823:cl_io_loop()) Process entered 00000100:00000001:1.0:1775768121.010181:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1775768121.010183:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000100:00000001:3.0:1775768121.010183:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775768121.010184:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00020000:00000001:0.0:1775768121.010186:0:13471:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00000100:00000001:1.0:1775768121.010187:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775768121.010188:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00020000:00200000:0.0:1775768121.010189:0:13471:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00080000:1.0:1775768121.010190:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00020000:00000002:0.0:1775768121.010194:0:13471:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00000100:00080000:1.0:1775768121.010197:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000040:3.0:1775768121.010197:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000067a0ba74] to pc [ptlrpcd_rcv+-1] req@ffff9942099ab480 x1862022294276864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:0.0:1775768121.010198:0:13471:0:(lov_io.c:155:lov_sub_get()) Process entered 00020000:00000001:0.0:1775768121.010200:0:13471:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000100:00000001:1.0:1775768121.010202:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000001:0.0:1775768121.010203:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:0.0:1775768121.010206:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff9942103be000. 00000020:00000010:0.0:1775768121.010211:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0e000. 00000100:00000040:1.0:1775768121.010211:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a8380 x1862022294276480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768146 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1775768121.010216:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210b0ea00. 00000100:00000001:3.0:1775768121.010217:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1775768121.010220:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942078a30c0. 00000100:00080000:3.0:1775768121.010223:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000008:00000010:0.0:1775768121.010225:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff9942086e4e18. 00020000:00000010:0.0:1775768121.010229:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994209592e40. 00000100:00080000:3.0:1775768121.010232:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00008000:00000010:0.0:1775768121.010234:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994220358000. 00000100:00100000:1.0:1775768121.010236:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294276480:192.168.202.156@tcp:250:kworker.0 00000100:00000001:3.0:1775768121.010236:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000080:00000010:0.0:1775768121.010238:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994207e3e3f0. 00000100:00080000:3.0:1775768121.010239:0:36:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000080:00000010:0.0:1775768121.010242:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b0ff8. 00000100:00000001:1.0:1775768121.010245:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:0.0:1775768121.010246:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0fa00. 00000100:00000001:3.0:1775768121.010246:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775768121.010250:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000008:00000010:0.0:1775768121.010251:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff99420906a6e0. 00000100:00080000:3.0:1775768121.010252:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5373 00020000:00000010:0.0:1775768121.010254:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff9942103cc480. 00000100:00000040:1.0:1775768121.010256:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a8380 x1862022294276480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768146 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00008000:00000010:0.0:1775768121.010258:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076adb0. 00000080:00000010:0.0:1775768121.010262:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff994210769870. 00000400:00000200:3.0:1775768121.010263:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:0.0:1775768121.010266:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107752091656 : -112965957459960 : ffff9942103be008) 00000020:00001000:0.0:1775768121.010272:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff9942103be000 00000400:00000200:3.0:1775768121.010272:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000020:00000001:0.0:1775768121.010275:0:13471:0:(cl_object.c:151:cl_object_top()) ffff9942078a75f8 -> ffff9942078a7f58 00000100:00080000:3.0:1775768121.010278:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000020:00000001:0.0:1775768121.010279:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:0.0:1775768121.010283:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1775768121.010286:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00000040:3.0:1775768121.010286:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768121.010287:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1775768121.010290:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768121.010291:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1775768121.010291:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000010:1.0:1775768121.010294:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543d000. 00020000:00000001:0.0:1775768121.010296:0:13471:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768121.010296:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000001:1.0:1775768121.010297:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768121.010300:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00020000:00000001:0.0:1775768121.010301:0:13471:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107752153128 : -112965957398488 : ffff9942103cd028) 00000020:00000040:1.0:1775768121.010302:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000100:00000001:3.0:1775768121.010302:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775768121.010306:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 00000020:00000001:0.0:1775768121.010308:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 02000000:00000010:1.0:1775768121.010308:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b1000. 00000020:00000001:3.0:1775768121.010308:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000001:0.0:1775768121.010312:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775768121.010312:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 02000000:00000001:1.0:1775768121.010314:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000020:00000001:3.0:1775768121.010315:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000001:1.0:1775768121.010316:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00020000:00000001:0.0:1775768121.010317:0:13471:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775768121.010319:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a8380. 00000020:00000040:3.0:1775768121.010319:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:0.0:1775768121.010320:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768121.010324:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000020:00000001:0.0:1775768121.010325:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:3.0:1775768121.010325:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000001:1.0:1775768121.010326:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00020000:00000001:0.0:1775768121.010328:0:13471:0:(lov_io.c:1108:lov_io_lock()) Process entered 00000100:00000001:1.0:1775768121.010330:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1775768121.010331:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00020000:00000001:0.0:1775768121.010332:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:0.0:1775768121.010335:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00000100:00000001:1.0:1775768121.010335:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:0.0:1775768121.010338:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000100:00000001:1.0:1775768121.010338:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768121.010338:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000001:0.0:1775768121.010341:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768121.010342:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775768121.010344:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:0.0:1775768121.010345:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768121.010346:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00020000:00000001:0.0:1775768121.010349:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768121.010349:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775768121.010349:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00020000:00000001:0.0:1775768121.010353:0:13471:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768121.010354:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000040:1.0:1775768121.010355:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a8700 x1862022294276608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775768121.010356:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:0.0:1775768121.010358:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775768121.010360:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000020:00000001:0.0:1775768121.010362:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768121.010367:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00000100:00080000:3.0:1775768121.010368:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00020000:00000001:0.0:1775768121.010369:0:13471:0:(lov_io.c:1114:lov_io_start()) Process entered 00020000:00000001:0.0:1775768121.010372:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:0.0:1775768121.010374:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00000100:00000001:3.0:1775768121.010375:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1775768121.010377:0:13471:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000100:00000001:1.0:1775768121.010377:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000008:00000001:0.0:1775768121.010380:0:13471:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000100:00000001:1.0:1775768121.010380:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768121.010380:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775768121.010384:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000001:0.0:1775768121.010385:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 02000000:00000001:1.0:1775768121.010386:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775768121.010386:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00000020:0.0:1775768121.010387:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:0.0:1775768121.010389:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1775768121.010392:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1775768121.010393:0:13471:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 02000000:00000001:3.0:1775768121.010393:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1775768121.010396:0:13471:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000100:00100000:1.0:1775768121.010397:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294276608:192.168.202.156@tcp:38:kworker.0 00000008:00000001:0.0:1775768121.010399:0:13471:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107884688256 : -112965824863360 : ffff994218232380) 00000100:00000001:3.0:1775768121.010399:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775768121.010402:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000008:00000020:0.0:1775768121.010403:0:13471:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994218232380 ready 0|-|- wr 1|-|+ rd 0|- 9 in flight 00000008:00100000:3.0:1775768121.010403:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 02000000:00000001:1.0:1775768121.010404:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768121.010407:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1775768121.010408:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 02000000:00000001:1.0:1775768121.010410:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:3.0:1775768121.010410:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000008:00000020:0.0:1775768121.010411:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:0.0:1775768121.010413:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1775768121.010415:0:13471:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 02000000:00000010:1.0:1775768121.010415:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543d000. 00000100:00000001:3.0:1775768121.010415:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000008:00000020:0.0:1775768121.010418:0:13471:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994210674fc0 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 02000000:00000001:1.0:1775768121.010419:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775768121.010422:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8e00. 00000008:00000001:0.0:1775768121.010425:0:13471:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1775768121.010426:0:36:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00000400:00000200:1.0:1775768121.010428:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00000001:0.0:1775768121.010429:0:13471:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768121.010433:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768121.010433:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000010:1.0:1775768121.010434:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 02000000:00000001:3.0:1775768121.010436:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768121.010437:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775768121.010438:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00020000:00000001:0.0:1775768121.010441:0:13471:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768121.010441:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000200:1.0:1775768121.010442:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294276608, portal 10 00000020:00000001:0.0:1775768121.010445:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768121.010445:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1775768121.010446:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768121.010448:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640854272 : -112966068697344 : ffff9942099a8700) 00000020:00000001:0.0:1775768121.010449:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 02000000:00000001:3.0:1775768121.010450:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768121.010452:0:13471:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00020000:00000001:0.0:1775768121.010454:0:13471:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000020:00000001:0.0:1775768121.010456:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00000100:00000040:1.0:1775768121.010457:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a8700 x1862022294276608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768146 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1775768121.010457:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0400. 00000020:00000001:0.0:1775768121.010458:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00020000:00000001:0.0:1775768121.010460:0:13471:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768121.010463:0:13471:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000100:00000001:3.0:1775768121.010464:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768121.010466:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00000020:00000001:0.0:1775768121.010468:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00020000:00000001:0.0:1775768121.010470:0:13471:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00020000:00000001:0.0:1775768121.010473:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:0.0:1775768121.010475:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000100:00000001:1.0:1775768121.010476:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:3.0:1775768121.010476:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8e00 x1862022294276992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000001:0.0:1775768121.010477:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:0.0:1775768121.010479:0:13471:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768121.010482:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00020000:00000001:0.0:1775768121.010483:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775768121.010486:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00020000:00000001:0.0:1775768121.010487:0:13471:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000020:00000001:0.0:1775768121.010489:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:0.0:1775768121.010491:0:13471:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000100:00000200:1.0:1775768121.010491:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294276608, offset 0 00000020:00000001:0.0:1775768121.010494:0:13471:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000001:0.0:1775768121.010496:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:0.0:1775768121.010498:0:13471:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00000400:00000200:1.0:1775768121.010499:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00020000:00000001:0.0:1775768121.010500:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:0.0:1775768121.010502:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000001:0.0:1775768121.010504:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000100:00000001:3.0:1775768121.010505:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00020000:00000001:0.0:1775768121.010506:0:13471:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768121.010507:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:0.0:1775768121.010508:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768121.010510:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768121.010512:0:13471:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000020:00000001:0.0:1775768121.010514:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000400:00000200:1.0:1775768121.010515:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000020:00000001:0.0:1775768121.010517:0:13471:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768121.010518:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000864a6866] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8e00 x1862022294276992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775768121.010519:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:0.0:1775768121.010520:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1775768121.010522:0:13471:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:0.0:1775768121.010524:0:13471:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:0.0:1775768121.010526:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00000400:00000200:1.0:1775768121.010526:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000020:00000001:0.0:1775768121.010529:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1775768121.010531:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff9942103be000 00000400:00000200:1.0:1775768121.010532:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000020:00000010:0.0:1775768121.010535:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210b0ea00. 00000400:00000200:1.0:1775768121.010537:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000020:00000010:0.0:1775768121.010541:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942078a30c0. 00000008:00000010:0.0:1775768121.010545:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff9942086e4e18. 00000400:00000200:1.0:1775768121.010545:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225334:225334:256:4294967295] 192.168.202.156@tcp LPNI seq info [225330:225330:8:4294967295] 00000100:00000001:3.0:1775768121.010545:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:0.0:1775768121.010551:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994209592e40. 00000400:00000200:1.0:1775768121.010553:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000040:3.0:1775768121.010553:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5383) 00008000:00000010:0.0:1775768121.010555:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994220358000. 00000080:00000010:0.0:1775768121.010560:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994207e3e3f0. 00000080:00000010:0.0:1775768121.010564:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b0ff8. 00000400:00000200:1.0:1775768121.010564:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000020:00000010:0.0:1775768121.010569:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0e000. 00000008:00000010:0.0:1775768121.010573:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff99420906a6e0. 00000800:00000200:1.0:1775768121.010573:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00020000:00000010:0.0:1775768121.010576:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff9942103cc480. 00000800:00000010:1.0:1775768121.010577:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00008000:00000010:0.0:1775768121.010579:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076adb0. 00000080:00000010:0.0:1775768121.010583:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff994210769870. 00000800:00000200:1.0:1775768121.010583:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768121.010587:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000020:00000010:0.0:1775768121.010588:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0fa00. 00000020:00000010:0.0:1775768121.010592:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff9942103be000. 00000800:00000200:1.1:1775768121.010595:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00020000:00000001:0.0:1775768121.010596:0:13471:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:0.0:1775768121.010598:0:13471:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:0.0:1775768121.010599:0:13471:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000080:00200000:0.0:1775768121.010600:0:13471:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:0.0:1775768121.010602:0:13471:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1775768121.010602:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1775768121.010603:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff9942103bed80 00000020:00000010:0.0:1775768121.010605:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210b0f600. 00000020:00000010:0.0:1775768121.010606:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942078a3d80. 00000800:00000200:1.1:1775768121.010609:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000008:00000010:0.0:1775768121.010610:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff9942086e6468. 00020000:00000010:0.0:1775768121.010615:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994209592f00. 00000800:00000100:1.0:1775768121.010619:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00008000:00000010:0.0:1775768121.010620:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff99422035b880. 00000800:00000010:1.0:1775768121.010622:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000080:00000010:0.0:1775768121.010623:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994207e3e150. 00000400:00000200:1.0:1775768121.010626:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000080:00000010:0.0:1775768121.010627:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b2238. 00000020:00000010:0.0:1775768121.010632:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0ec00. 00000400:00000200:1.0:1775768121.010632:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000008:00000010:0.0:1775768121.010638:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff99420906be40. 00000400:00000200:1.0:1775768121.010638:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768121.010641:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00020000:00000010:0.0:1775768121.010642:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff9942103ccfc0. 00000100:00000001:1.0:1775768121.010645:0:2399:0:(events.c:34:request_out_callback()) Process entered 00008000:00000010:0.0:1775768121.010647:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076afa0. 00000100:00000200:1.0:1775768121.010648:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a8700 x1862022294276608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768146 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000010:0.0:1775768121.010652:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942107690f0. 00000020:00000010:0.0:1775768121.010657:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0fc00. 00000100:00000001:1.0:1775768121.010658:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000020:00000010:0.0:1775768121.010662:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff9942103bed80. 00000100:00000040:1.0:1775768121.010663:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a8700 x1862022294276608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768146 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000001:0.0:1775768121.010668:0:13471:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768121.010671:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1775768121.010672:0:13471:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768121.010676:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000080:00000001:0.0:1775768121.010679:0:13471:0:(rw.c:1481:ll_writepages()) Process entered 00000080:00000001:0.0:1775768121.010681:0:13471:0:(file.c:5250:cl_sync_file_range()) Process entered 00000020:00000001:0.0:1775768121.010682:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00000001:1.0:1775768121.010682:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1775768121.010686:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff9942103bed80. 00000100:00000040:1.0:1775768121.010687:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000020:00000010:0.0:1775768121.010689:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0fc00. 00000020:00000010:0.0:1775768121.010692:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210b0ec00. 00000400:00000200:1.0:1775768121.010695:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000010:0.0:1775768121.010696:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942078a3d80. 00000008:00000010:0.0:1775768121.010700:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff9942086e6468. 00000100:00000001:1.0:1775768121.010700:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300045843 00020000:00000010:0.0:1775768121.010703:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994209592f00. 00000100:00000001:1.0:1775768121.010705:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300047165 00008000:00000010:0.0:1775768121.010706:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff99422035b880. 00000080:00000010:0.0:1775768121.010711:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994207e3e150. 00000100:00000001:1.0:1775768121.010711:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300045843 00000080:00000010:0.0:1775768121.010715:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b2238. 00000100:00000001:1.0:1775768121.010716:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300047165 00000020:00000010:0.0:1775768121.010719:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0f600. 00000100:00000001:1.0:1775768121.010720:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00000010:0.0:1775768121.010722:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff99420906be40. 00000100:00000001:1.0:1775768121.010736:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:0.0:1775768121.010737:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff9942103ccfc0. 00008000:00000010:0.0:1775768121.010739:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076afa0. 00000100:00000001:1.0:1775768121.010741:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000080:00000010:0.0:1775768121.010742:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942107690f0. 00000020:00000001:0.0:1775768121.010744:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107752095112 : -112965957456504 : ffff9942103bed88) 00000020:00001000:0.0:1775768121.010747:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff9942103bed80 00000100:00100000:1.0:1775768121.010748:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768121/real 1775768121] req@ffff9942099a8700 x1862022294276608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775768121.010751:0:13471:0:(cl_object.c:151:cl_object_top()) ffff99420791a850 -> ffff99420791a850 00000020:00000001:0.0:1775768121.010755:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1775768121.010757:0:13471:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:0.0:1775768121.010759:0:13471:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017f:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1775768121.010762:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000400:00000001:1.0:1775768121.010762:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:0.0:1775768121.010764:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1775768121.010765:0:13471:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1775768121.010765:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768121.010767:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00020000:00000002:0.0:1775768121.010768:0:13471:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017f:0x0]io ffff9942077b2298 type 6 ignore/verify layout 1/0 00020000:00000002:0.0:1775768121.010772:0:13471:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207dfbd90) owner 0000000000000000/ffff994220080000 00000400:00000200:1.0:1775768121.010772:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768121.010775:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00020000:00000001:0.0:1775768121.010776:0:13471:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1775768121.010779:0:13471:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00000100:00000001:1.0:1775768121.010779:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00020000:00000001:0.0:1775768121.010780:0:13471:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00020000:00000001:0.0:1775768121.010782:0:13471:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775768121.010784:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a8700 x1862022294276608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:0.0:1775768121.010785:0:13471:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1775768121.010788:0:13471:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1775768121.010790:0:13471:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768121.010794:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1775768121.010796:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1775768121.010798:0:13471:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00020000:00000002:0.0:1775768121.010800:0:13471:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207dfbd90) owner 0000000000000000/ffff994220080000 00000020:00000001:0.0:1775768121.010803:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768121.010806:0:13471:0:(cl_io.c:823:cl_io_loop()) Process entered 00000100:00000200:1.0:1775768121.010806:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a8700 x1862022294276608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775768121.010807:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00020000:00000001:0.0:1775768121.010809:0:13471:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00020000:00200000:0.0:1775768121.010811:0:13471:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00000002:0.0:1775768121.010814:0:13471:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:0.0:1775768121.010819:0:13471:0:(lov_io.c:155:lov_sub_get()) Process entered 00020000:00000001:0.0:1775768121.010821:0:13471:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000020:00000001:0.0:1775768121.010823:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:0.0:1775768121.010825:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff9942103be000. 00000020:00000010:0.0:1775768121.010829:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0fa00. 00000020:00000010:0.0:1775768121.010833:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210b0e000. 00000100:00000001:1.0:1775768121.010835:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000010:0.0:1775768121.010836:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942078a30c0. 00000008:00000010:0.0:1775768121.010840:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff9942086e4e18. 00000100:00000001:1.0:1775768121.010840:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00020000:00000010:0.0:1775768121.010842:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994209592e40. 00000100:00000001:1.0:1775768121.010843:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00008000:00000010:0.0:1775768121.010844:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994220358000. 00000100:00000001:1.0:1775768121.010846:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000080:00000010:0.0:1775768121.010847:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994207e3e3f0. 00000080:00000010:0.0:1775768121.010850:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b0ff8. 00000100:00100000:1.0:1775768121.010853:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a8700 x1862022294276608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1775768121.010854:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0ea00. 00000008:00000010:0.0:1775768121.010857:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff99420906a6e0. 00020000:00000010:0.0:1775768121.010860:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff9942103cc480. 00008000:00000010:0.0:1775768121.010864:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076adb0. 00000080:00000010:0.0:1775768121.010868:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff994210769870. 00000100:00000001:1.0:1775768121.010869:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1775768121.010872:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107752091656 : -112965957459960 : ffff9942103be008) 00000100:00000040:1.0:1775768121.010875:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a8700 x1862022294276608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768146 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00001000:0.0:1775768121.010876:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff9942103be000 00000020:00000001:0.0:1775768121.010879:0:13471:0:(cl_object.c:151:cl_object_top()) ffff99420791a210 -> ffff99420791a850 00000020:00000001:0.0:1775768121.010882:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:0.0:1775768121.010884:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1775768121.010886:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:0.0:1775768121.010888:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768121.010891:0:13471:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768121.010894:0:13471:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107752153128 : -112965957398488 : ffff9942103cd028) 00000100:00000001:1.0:1775768121.010894:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:0.0:1775768121.010898:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000020:00000001:0.0:1775768121.010900:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768121.010900:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00020000:00000001:0.0:1775768121.010901:0:13471:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768121.010903:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768121.010903:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768121.010905:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1775768121.010906:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00020000:00000001:0.0:1775768121.010907:0:13471:0:(lov_io.c:1108:lov_io_lock()) Process entered 00020000:00000001:0.0:1775768121.010910:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000001:1.0:1775768121.010910:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000020:00000001:0.0:1775768121.010911:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:0.0:1775768121.010913:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:0.0:1775768121.010914:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768121.010915:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:0.0:1775768121.010916:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768121.010918:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768121.010920:0:13471:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768121.010920:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000020:00000001:0.0:1775768121.010923:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:0.0:1775768121.010925:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768121.010927:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768121.010928:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000020:00000001:0.0:1775768121.010930:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00020000:00000001:0.0:1775768121.010932:0:13471:0:(lov_io.c:1114:lov_io_start()) Process entered 00000100:00000001:1.0:1775768121.010933:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00020000:00000001:0.0:1775768121.010934:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:0.0:1775768121.010935:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00000008:00000001:0.0:1775768121.010937:0:13471:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000008:00000001:0.0:1775768121.010939:0:13471:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000100:00000040:1.0:1775768121.010940:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a8700 x1862022294276608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768146 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:0.0:1775768121.010941:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:0.0:1775768121.010943:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:0.0:1775768121.010945:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1775768121.010950:0:13471:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000008:00000001:0.0:1775768121.010953:0:13471:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000008:00000001:0.0:1775768121.010955:0:13471:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107754938304 : -112965954613312 : ffff994210674fc0) 00000100:00100000:1.0:1775768121.010956:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294276608:192.168.202.156@tcp:38:kworker.0 00000008:00000020:0.0:1775768121.010960:0:13471:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994210674fc0 ready 0|-|- wr 1|-|+ rd 0|- 9 in flight 00000100:00000001:1.0:1775768121.010962:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000008:00000001:0.0:1775768121.010965:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:0.0:1775768121.010968:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000100:00000040:1.0:1775768121.010968:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a9180 x1862022294276736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:0.0:1775768121.010971:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1775768121.010975:0:13471:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000008:00000020:0.0:1775768121.010978:0:13471:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994218232380 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000100:00000001:1.0:1775768121.010984:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000008:00000001:0.0:1775768121.010987:0:13471:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768121.010987:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1775768121.010992:0:13471:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768121.010993:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768121.010995:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1775768121.010996:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768121.010999:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768121.010999:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768121.011003:0:13471:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768121.011006:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768121.011006:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294276736:192.168.202.156@tcp:38:kworker.0 00000020:00000001:0.0:1775768121.011010:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00020000:00000001:0.0:1775768121.011012:0:13471:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00020000:00000001:0.0:1775768121.011015:0:13471:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000100:00000001:1.0:1775768121.011015:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000001:0.0:1775768121.011018:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000001:0.0:1775768121.011019:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 02000000:00000001:1.0:1775768121.011019:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00020000:00000001:0.0:1775768121.011021:0:13471:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768121.011022:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768121.011023:0:13471:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000020:00000001:0.0:1775768121.011025:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 02000000:00000001:1.0:1775768121.011027:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:0.0:1775768121.011028:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00020000:00000001:0.0:1775768121.011030:0:13471:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00020000:00000001:0.0:1775768121.011031:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:0.0:1775768121.011031:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000020:00000001:0.0:1775768121.011032:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:0.0:1775768121.011033:0:13471:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775768121.011033:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543d400. 00020000:00000001:0.0:1775768121.011036:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768121.011038:0:13471:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 02000000:00000001:1.0:1775768121.011038:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768121.011040:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:0.0:1775768121.011042:0:13471:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000020:00000001:0.0:1775768121.011043:0:13471:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000001:0.0:1775768121.011045:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000400:00000200:1.0:1775768121.011045:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:0.0:1775768121.011048:0:13471:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00000400:00000010:1.0:1775768121.011049:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00020000:00000001:0.0:1775768121.011050:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:0.0:1775768121.011052:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000001:0.0:1775768121.011054:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000400:00000010:1.0:1775768121.011055:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00020000:00000001:0.0:1775768121.011056:0:13471:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768121.011060:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775768121.011061:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294276736, portal 10 00020000:00000001:0.0:1775768121.011063:0:13471:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000100:00000001:1.0:1775768121.011064:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000020:00000001:0.0:1775768121.011065:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000100:00000001:1.0:1775768121.011067:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640856960 : -112966068694656 : ffff9942099a9180) 00000020:00000001:0.0:1775768121.011068:0:13471:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768121.011072:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1775768121.011074:0:13471:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:0.0:1775768121.011077:0:13471:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000100:00000040:1.0:1775768121.011077:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a9180 x1862022294276736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768146 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775768121.011079:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:0.0:1775768121.011081:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1775768121.011084:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff9942103be000 00000020:00000010:0.0:1775768121.011089:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210b0e000. 00000100:00000001:1.0:1775768121.011090:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000010:0.0:1775768121.011094:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942078a30c0. 00000100:00000040:1.0:1775768121.011097:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000008:00000010:0.0:1775768121.011099:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff9942086e4e18. 00000400:00000010:1.0:1775768121.011100:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00020000:00000010:0.0:1775768121.011103:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994209592e40. 00000100:00000200:1.0:1775768121.011105:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294276736, offset 0 00008000:00000010:0.0:1775768121.011107:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994220358000. 00000080:00000010:0.0:1775768121.011112:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994207e3e3f0. 00000400:00000200:1.0:1775768121.011112:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000080:00000010:0.0:1775768121.011118:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b0ff8. 00000400:00000200:1.0:1775768121.011120:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000010:0.0:1775768121.011124:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0fa00. 00000400:00000200:1.0:1775768121.011125:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000008:00000010:0.0:1775768121.011130:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff99420906a6e0. 00000400:00000200:1.0:1775768121.011130:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768121.011134:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00020000:00000010:0.0:1775768121.011135:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff9942103cc480. 00000400:00000200:1.0:1775768121.011137:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00008000:00000010:0.0:1775768121.011139:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076adb0. 00000400:00000200:1.0:1775768121.011141:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000080:00000010:0.0:1775768121.011144:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff994210769870. 00000400:00000200:1.0:1775768121.011145:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225335:225335:256:4294967295] 192.168.202.156@tcp LPNI seq info [225331:225331:8:4294967295] 00000020:00000010:0.0:1775768121.011150:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0ea00. 00000400:00000200:1.0:1775768121.011150:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768121.011178:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000020:00000010:0.0:1775768121.011179:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff9942103be000. 00000800:00000200:1.0:1775768121.011184:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00020000:00000001:0.0:1775768121.011185:0:13471:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00000800:00000010:1.0:1775768121.011188:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00020000:00000001:0.0:1775768121.011189:0:13471:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:0.0:1775768121.011192:0:13471:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000800:00000200:1.0:1775768121.011195:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000080:00200000:0.0:1775768121.011196:0:13471:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017f:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000800:00000200:1.0:1775768121.011198:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000080:00000001:0.0:1775768121.011201:0:13471:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1775768121.011203:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000800:00000200:1.1:1775768121.011205:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00001000:0.0:1775768121.011207:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff9942103bed80 00000800:00000200:1.1:1775768121.011208:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000020:00000010:0.0:1775768121.011212:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210b0ec00. 00000800:00000100:1.0:1775768121.011214:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768121.011217:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000020:00000010:0.0:1775768121.011218:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942078a3d80. 00000400:00000200:1.0:1775768121.011219:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000008:00000010:0.0:1775768121.011222:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff9942086e6468. 00000400:00000200:1.0:1775768121.011223:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768121.011228:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00020000:00000010:0.0:1775768121.011229:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994209592f00. 00000400:00000010:1.0:1775768121.011231:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00008000:00000010:0.0:1775768121.011233:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff99422035b880. 00000100:00000001:1.0:1775768121.011234:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000080:00000010:0.0:1775768121.011239:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994207e3e150. 00000100:00000200:1.0:1775768121.011239:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a9180 x1862022294276736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768146 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000010:0.0:1775768121.011244:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b2238. 00000020:00000010:0.0:1775768121.011250:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0fc00. 00000100:00000001:1.0:1775768121.011251:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000008:00000010:0.0:1775768121.011255:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff99420906be40. 00000100:00000040:1.0:1775768121.011255:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a9180 x1862022294276736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768146 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000010:0.0:1775768121.011261:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff9942103ccfc0. 00000100:00000001:1.0:1775768121.011263:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768121.011265:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00008000:00000010:0.0:1775768121.011266:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076afa0. 00000100:00000001:1.0:1775768121.011268:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1775768121.011270:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942107690f0. 00000100:00000040:1.0:1775768121.011271:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000020:00000010:0.0:1775768121.011275:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0f600. 00000400:00000200:1.0:1775768121.011275:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768121.011278:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300045836 00000020:00000010:0.0:1775768121.011280:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff9942103bed80. 00000100:00000001:1.0:1775768121.011280:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300047165 00000080:00000001:0.0:1775768121.011284:0:13471:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768121.011284:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300045836 00000100:00000001:1.0:1775768121.011286:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300047165 00000080:00000001:0.0:1775768121.011289:0:13471:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768121.011289:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768121.011294:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768121.011296:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768121.011298:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768121/real 1775768121] req@ffff9942099a9180 x1862022294276736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768121.011303:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768121.011304:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768121.011305:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768121.011307:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768121.011307:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768121.011309:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768121.011311:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a9180 x1862022294276736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768121.011316:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a9180 x1862022294276736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768121.011320:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768121.011320:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768121.011321:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768121.011322:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768121.011324:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a9180 x1862022294276736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768121.011327:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768121.011329:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a9180 x1862022294276736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768146 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768121.011333:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768121.011334:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768121.011335:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768121.011335:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768121.011336:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768121.011338:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768121.011339:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768121.011341:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768121.011342:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768121.011344:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a9180 x1862022294276736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768146 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768121.011349:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294276736:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768121.011351:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768121.011353:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ab480 x1862022294276864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768121.011356:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768121.011357:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768121.011358:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768121.011359:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768121.011360:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768121.011362:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ab480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294276864:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768121.011364:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768121.011365:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768121.011366:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768121.011367:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768121.011369:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543e400. 02000000:00000001:1.0:1775768121.011370:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768121.011372:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768121.011373:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000400:00000010:1.0:1775768121.011375:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000200:1.0:1775768121.011377:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294276864, portal 4 00000100:00000001:1.0:1775768121.011378:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768121.011379:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640865920 : -112966068685696 : ffff9942099ab480) 00000100:00000040:1.0:1775768121.011382:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ab480 x1862022294276864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768146 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768121.011385:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768121.011387:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768121.011388:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768121.011389:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294276864, offset 0 00000400:00000200:1.0:1775768121.011392:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768121.011394:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768121.011396:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768121.011397:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768121.011399:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768121.011400:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768121.011402:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768121.011404:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225336:225336:256:4294967295] 192.168.202.156@tcp LPNI seq info [225332:225332:8:4294967295] 00000400:00000200:1.0:1775768121.011406:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768121.011409:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768121.011412:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768121.011413:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000800:00000200:1.0:1775768121.011415:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768121.011417:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775768121.011419:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768121.011421:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000100:1.0:1775768121.011423:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768121.011424:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768121.011425:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768121.011427:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768121.011429:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768121.011430:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768121.011431:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768121.011433:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ab480 x1862022294276864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768146 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768121.011437:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768121.011439:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ab480 x1862022294276864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768146 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768121.011442:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768121.011443:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768121.011444:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768121.011446:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:1.0:1775768121.011447:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768121.011449:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300045859 00000100:00000001:1.0:1775768121.011450:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300047165 00000100:00000001:1.0:1775768121.011451:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300045859 00000100:00000001:1.0:1775768121.011452:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300047165 00000100:00000001:1.0:1775768121.011453:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768121.011454:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768121.011455:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768121.011457:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768121/real 1775768121] req@ffff9942099ab480 x1862022294276864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768121.011461:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768121.011462:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768121.011462:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768121.011464:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768121.011465:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768121.011466:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768121.011467:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ab480 x1862022294276864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768121.011471:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ab480 x1862022294276864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768121.011475:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768121.011475:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768121.011476:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768121.011477:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768121.011478:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ab480 x1862022294276864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768121.011482:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768121.011484:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ab480 x1862022294276864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768146 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768121.011487:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768121.011489:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768121.011490:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768121.011490:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768121.011491:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768121.011494:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768121.011495:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768121.011497:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768121.011498:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768121.011500:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ab480 x1862022294276864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768146 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768121.011504:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ab480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294276864:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768121.011507:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768121.011509:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768121.011511:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a8700 x1862022294276608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768146 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768121.011517:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768121.011519:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768121.011520:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543d000. 02000000:00000001:1.0:1775768121.011522:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768121.011523:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768121.011525:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:1.0:1775768121.011526:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768121.011528:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b0c00. 02000000:00000001:1.0:1775768121.011530:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768121.011532:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768121.011533:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a8700. 00000100:00000001:1.0:1775768121.011535:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768121.011536:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768121.011538:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768121.011541:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a9180 x1862022294276736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768146 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768121.011553:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768121.011555:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768121.011556:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543d400. 02000000:00000001:1.0:1775768121.011558:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768121.011559:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768121.011560:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:1.0:1775768121.011561:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768121.011563:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b2000. 02000000:00000001:1.0:1775768121.011564:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768121.011565:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768121.011566:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a9180. 00000100:00000001:1.0:1775768121.011567:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768121.011568:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768121.011569:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768121.011571:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ab480 x1862022294276864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768146 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768121.011575:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768121.011576:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768121.011576:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543e400. 02000000:00000001:1.0:1775768121.011578:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768121.011579:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768121.011580:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:1.0:1775768121.011582:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768121.011583:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b3400. 02000000:00000001:1.0:1775768121.011585:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768121.011586:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768121.011587:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ab480. 00000100:00000001:1.0:1775768121.011589:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768121.011590:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768121.011592:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768121.011595:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768121.011596:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768121.011598:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768121.011599:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768121.011601:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768121.011605:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a8e00 x1862022294276992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768121.011611:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768121.011613:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768121.011615:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768121.011616:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768121.011618:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768121.011621:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294276992:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768121.011624:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768121.011625:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768121.011627:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768121.011628:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768121.011630:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543e400. 02000000:00000001:1.0:1775768121.011631:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768121.011633:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768121.011635:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000400:00000010:1.0:1775768121.011637:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000200:1.0:1775768121.011639:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294276992, portal 4 00000100:00000001:1.0:1775768121.011640:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768121.011641:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640856064 : -112966068695552 : ffff9942099a8e00) 00000100:00000040:1.0:1775768121.011644:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a8e00 x1862022294276992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768146 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768121.011648:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768121.011649:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768121.011651:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768121.011652:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294276992, offset 0 00000400:00000200:1.0:1775768121.011654:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768121.011656:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768121.011658:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768121.011660:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768121.011661:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768121.011662:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768121.011664:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768121.011666:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225337:225337:256:4294967295] 192.168.202.156@tcp LPNI seq info [225333:225333:8:4294967295] 00000400:00000200:1.0:1775768121.011669:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768121.011674:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768121.011679:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768121.011682:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000800:00000200:1.0:1775768121.011685:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768121.011688:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775768121.011691:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768121.011693:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000100:1.0:1775768121.011697:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768121.011698:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768121.011701:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768121.011704:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768121.011706:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768121.011708:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768121.011710:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768121.011713:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a8e00 x1862022294276992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768146 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768121.011721:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768121.011739:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a8e00 x1862022294276992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768146 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768121.011744:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768121.011745:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768121.011747:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768121.011749:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:1.0:1775768121.011752:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768121.011753:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300045862 00000100:00000001:1.0:1775768121.011754:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300047165 00000100:00000001:1.0:1775768121.011756:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300045862 00000100:00000001:1.0:1775768121.011757:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300047165 00000100:00000001:1.0:1775768121.011759:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768121.011760:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768121.011761:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768121.011764:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768121/real 1775768121] req@ffff9942099a8e00 x1862022294276992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768121.011768:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768121.011769:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768121.011770:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768121.011772:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768121.011772:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768121.011774:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768121.011775:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a8e00 x1862022294276992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768121.011780:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a8e00 x1862022294276992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768121.011784:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768121.011785:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768121.011787:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768121.011788:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768121.011791:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a8e00 x1862022294276992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768121.011798:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768121.011801:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a8e00 x1862022294276992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768146 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768121.011808:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768121.011810:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768121.011811:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768121.011813:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768121.011814:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768121.011818:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768121.011820:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768121.011823:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768121.011825:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768121.011829:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a8e00 x1862022294276992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768146 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768121.011834:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294276992:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768121.011836:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768121.011837:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768121.011839:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a8e00 x1862022294276992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768146 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768121.011842:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768121.011843:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768121.011844:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543e400. 02000000:00000001:1.0:1775768121.011845:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768121.011846:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768121.011847:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:1.0:1775768121.011848:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768121.011849:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b0400. 02000000:00000001:1.0:1775768121.011850:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768121.011851:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768121.011852:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a8e00. 00000100:00000001:1.0:1775768121.011853:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768121.011854:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768121.011855:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768121.011857:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768121.011858:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768121.011859:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768121.011860:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768121.073252:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768121.073256:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768121.073260:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768121.073261:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768121.073264:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768121.073265:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768121.073267:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768121.073268:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768121.073270:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768121.073271:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768121.073317:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768121.073320:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768121.073324:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768121.137268:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768121.137275:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768121.137277:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768121.137290:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768121.137293:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768121.137297:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768121.137299:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768121.137302:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768121.137304:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768122.033247:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768122.033247:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768122.033247:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768122.033253:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768122.033253:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768122.033254:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768122.033256:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768122.033256:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768122.033257:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768122.033262:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:0.0:1775768122.033263:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:2.0:1775768122.033263:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768122.033267:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768122.033268:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768122.033269:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768122.033270:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768122.033272:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768122.033272:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768122.033276:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768122.033279:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768122.033281:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768122.033282:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768122.033284:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768122.033287:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768122.033287:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768122.033290:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768122.033293:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768122.097242:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768122.097247:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768122.097249:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768122.097252:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768122.097255:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768122.097256:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768122.097258:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768122.097259:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768122.097261:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768122.097263:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768122.097265:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768122.097266:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768122.097268:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768122.097268:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768122.097271:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768122.097272:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768122.097324:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768122.097327:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768122.097330:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768122.161225:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768122.161230:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768122.161232:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768122.161236:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768122.161238:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768122.161241:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768122.161243:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768122.161244:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768122.161246:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768122.353214:0:14953:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4300047165, at 4300047168 00000100:00000010:0.0:1775768122.353225:0:14953:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cba060. 00000100:00000040:1.0:1775768122.353227:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4300047165, at 4300047168 00000100:00000010:1.0:1775768122.353236:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cba1e0. 00000100:00000001:0.0:1775768123.057256:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768123.057256:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768123.057257:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768123.057261:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768123.057262:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768123.057263:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768123.057264:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768123.057265:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768123.057270:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:2.0:1775768123.057271:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768123.057274:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768123.057275:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768123.057276:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1775768123.057280:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775768123.057281:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768123.057281:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768123.057286:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768123.057286:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768123.057290:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768123.057291:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768123.057292:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768123.057292:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768123.057296:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768123.057299:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768123.057301:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768123.057304:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768123.057307:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768123.121257:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768123.121263:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768123.121268:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768123.121270:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768123.121274:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768123.121275:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768123.121278:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768123.121280:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768123.121283:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768123.121284:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768123.121337:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768123.121341:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768123.121344:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768123.185253:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768123.185258:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768123.185259:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768123.185263:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768123.185266:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768123.185269:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768123.185271:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768123.185272:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768123.185274:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768124.081255:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768124.081255:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768124.081263:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768124.081263:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768124.081266:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768124.081267:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768124.081273:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775768124.081273:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768124.081281:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768124.081281:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768124.081285:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768124.081285:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768124.081290:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768124.081292:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768124.081295:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768124.081295:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768124.081299:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768124.081299:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768124.081302:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768124.081309:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768124.081312:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768124.081318:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768124.081322:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768124.081325:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768124.081328:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768124.081331:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768124.081333:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768124.145287:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768124.145295:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768124.145301:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768124.145303:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768124.145306:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768124.145308:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768124.145311:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768124.145312:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768124.145316:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768124.145317:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768124.145383:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768124.145389:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768124.145400:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768124.209310:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768124.209319:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768124.209322:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768124.209329:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768124.209334:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768124.209338:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768124.209341:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768124.209344:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768124.209347:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768125.105292:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768125.105292:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768125.105300:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768125.105302:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768125.105302:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768125.105307:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768125.105309:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768125.105317:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1775768125.105317:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775768125.105321:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768125.105324:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768125.105325:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768125.105327:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768125.105330:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768125.105331:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768125.105336:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768125.105338:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768125.105342:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768125.105442:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768125.105446:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768125.105447:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768125.105450:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768125.105452:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768125.105454:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768125.105456:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768125.105457:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768125.105459:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768125.169316:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768125.169324:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768125.169331:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768125.169332:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768125.169336:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768125.169338:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768125.169341:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768125.169342:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768125.169346:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768125.169347:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768125.169417:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768125.169422:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768125.169426:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768125.233304:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768125.233311:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768125.233312:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768125.233317:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768125.233321:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768125.233325:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768125.233327:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768125.233329:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768125.233332:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768126.129238:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768126.129238:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768126.129240:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768126.129244:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768126.129244:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768126.129245:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768126.129247:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768126.129248:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768126.129248:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768126.129252:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775768126.129255:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:1.0:1775768126.129256:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775768126.129256:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00080000:0.0:1775768126.129257:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775768126.129261:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775768126.129263:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768126.129263:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768126.129263:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1775768126.129264:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768126.129267:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768126.129267:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768126.129268:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775768126.129268:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768126.129272:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1775768126.129275:0:13471:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5378 00000100:00000001:1.0:1775768126.129275:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768126.129278:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768126.129279:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768126.129281:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768126.129284:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768126.129284:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768126.129285:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768126.129288:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768126.129292:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768126.129299:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768126.129302:0:13471:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768126.129306:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768126.129309:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768126.129311:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768126.129314:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768126.129317:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768126.129318:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:0.0:1775768126.129320:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768126.129322:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:0.0:1775768126.129323:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:0.0:1775768126.129325:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:0.0:1775768126.129328:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768126.129329:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768126.129330:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768126.129331:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768126.129333:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:0.0:1775768126.129335:0:13471:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768126.129336:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768126.129338:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768126.129341:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768126.129342:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768126.129343:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775768126.129345:0:13471:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768126.129351:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fe300. 00000020:00000040:0.0:1775768126.129353:0:13471:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:0.0:1775768126.129355:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768126.129356:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768126.129358:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768126.129361:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768126.129365:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768126.129371:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220028000. 00000100:00000001:0.0:1775768126.129377:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768126.129386:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fe300 x1862022294277120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768126.129401:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768126.129405:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768126.129409:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008a1e3639] to pc [ptlrpcd_rcv+-1] req@ffff9942099fe300 x1862022294277120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768126.129425:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768126.129427:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768126.129429:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775768126.129430:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768126.129431:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768126.129434:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768126.129435:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5378 00000400:00000200:0.0:1775768126.129439:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768126.129441:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:0.0:1775768126.129442:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768126.129445:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1775768126.129445:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768126.129448:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768126.129449:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768126.129451:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768126.129452:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768126.129453:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768126.129456:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768126.129457:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:0.0:1775768126.129458:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768126.129460:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000100:00000040:1.0:1775768126.129460:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fe300 x1862022294277120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775768126.129464:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:0.0:1775768126.129468:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:0.0:1775768126.129473:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768126.129475:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775768126.129475:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768126.129476:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768126.129478:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000001:1.0:1775768126.129480:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1775768126.129481:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 02000000:00000001:1.0:1775768126.129483:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:0.0:1775768126.129484:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 02000000:00000001:1.0:1775768126.129487:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768126.129488:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768126.129492:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768126.129493:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294277120:192.168.202.156@tcp:250:kworker.0 02000000:00000001:0.0:1775768126.129494:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1775768126.129497:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768126.129499:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768126.129500:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768126.129503:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768126.129503:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768126.129508:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:1.0:1775768126.129508:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:0.0:1775768126.129513:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 02000000:00000010:1.0:1775768126.129513:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543e400. 00000008:00000001:0.0:1775768126.129517:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768126.129517:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768126.129520:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768126.129523:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fc700. 00000400:00000200:1.0:1775768126.129523:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000040:0.0:1775768126.129526:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:0.0:1775768126.129529:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000010:1.0:1775768126.129529:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 02000000:00000001:0.0:1775768126.129532:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768126.129533:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768126.129534:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768126.129535:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775768126.129535:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 02000000:00000010:0.0:1775768126.129540:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002b800. 00000100:00000001:0.0:1775768126.129543:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775768126.129545:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294277120, portal 25 00000100:00100000:0.0:1775768126.129548:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fc700 x1862022294277248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775768126.129548:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768126.129550:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641205504 : -112966068346112 : ffff9942099fe300) 00000100:00000001:0.0:1775768126.129558:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:1.0:1775768126.129558:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fe300 x1862022294277120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768151 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768126.129562:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768126.129569:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003283b20a] to pc [ptlrpcd_rcv+-1] req@ffff9942099fc700 x1862022294277248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768126.129588:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1775768126.129590:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768126.129593:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000100:00080000:0.0:1775768126.129594:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:1.0:1775768126.129596:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00080000:0.0:1775768126.129597:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775768126.129600:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000200:1.0:1775768126.129600:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294277120, offset 0 00000100:00080000:0.0:1775768126.129602:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768126.129605:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:1.0:1775768126.129608:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00080000:0.0:1775768126.129610:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5378 00000400:00000200:0.0:1775768126.129613:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768126.129615:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768126.129616:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1775768126.129618:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000040:0.0:1775768126.129620:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768126.129623:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1775768126.129625:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00000040:0.0:1775768126.129626:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768126.129630:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000200:1.0:1775768126.129631:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:0.0:1775768126.129634:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000400:00000200:1.0:1775768126.129635:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000020:00000040:0.0:1775768126.129637:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:0.0:1775768126.129640:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:1.0:1775768126.129640:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000020:00000040:0.0:1775768126.129643:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000400:00000200:1.0:1775768126.129644:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000020:00000001:0.0:1775768126.129647:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000400:00000200:1.0:1775768126.129648:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225338:225338:256:4294967295] 192.168.202.156@tcp LPNI seq info [225334:225334:8:4294967295] 00000020:00000001:0.0:1775768126.129653:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000400:00000200:1.0:1775768126.129655:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000040:0.0:1775768126.129661:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768126.129665:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1775768126.129667:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000100:00000040:0.0:1775768126.129671:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000800:00000200:1.0:1775768126.129685:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768126.129689:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000100:00000001:0.0:1775768126.129691:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768126.129694:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000800:00000200:1.0:1775768126.129698:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00080000:0.0:1775768126.129699:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768126.129703:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768126.129704:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:1.0:1775768126.129704:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 02000000:00000001:0.0:1775768126.129707:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768126.129710:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768126.129713:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000800:00000200:1.1:1775768126.129714:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000008:00100000:0.0:1775768126.129715:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775768126.129718:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768126.129720:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775768126.129720:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000100:00000010:0.0:1775768126.129724:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ff100. 00000020:00000040:0.0:1775768126.129728:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000800:00000100:1.0:1775768126.129728:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 02000000:00000001:0.0:1775768126.129732:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000800:00000010:1.0:1775768126.129732:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 02000000:00000001:0.0:1775768126.129735:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768126.129738:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768126.129739:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:1.0:1775768126.129740:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000001:0.0:1775768126.129741:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768126.129747:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002bc00. 00000400:00000200:1.0:1775768126.129747:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000100:00000001:0.0:1775768126.129752:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768126.129753:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768126.129756:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00100000:0.0:1775768126.129760:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ff100 x1862022294277376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775768126.129760:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768126.129765:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fe300 x1862022294277120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768151 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768126.129780:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775768126.129782:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1775768126.129784:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768126.129786:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fe300 x1862022294277120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768151 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775768126.129790:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5e24158] to pc [ptlrpcd_rcv+-1] req@ffff9942099ff100 x1862022294277376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768126.129814:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768126.129815:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768126.129817:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775768126.129818:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:0.0:1775768126.129822:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775768126.129824:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775768126.129824:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768126.129826:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768126.129829:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:1.0:1775768126.129831:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00080000:0.0:1775768126.129834:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5378 00000100:00000001:1.0:1775768126.129836:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4300052285 00000400:00000200:0.0:1775768126.129841:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000010:1.0:1775768126.129843:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba1e0. 00000400:00000200:0.0:1775768126.129846:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768126.129849:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1775768126.129851:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000040:0.0:1775768126.129854:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768126.129857:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775768126.129858:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4300052285 00000100:00000040:0.0:1775768126.129860:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000010:1.0:1775768126.129863:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cbad80. 00000100:00000001:0.0:1775768126.129864:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768126.129868:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:1.0:1775768126.129868:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000020:00000040:0.0:1775768126.129872:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000100:00000001:1.0:1775768126.129873:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1775768126.129875:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768126.129878:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:0.0:1775768126.129881:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:1.0:1775768126.129883:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768126.129884:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:1.0:1775768126.129887:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000040:0.0:1775768126.129890:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00100000:1.0:1775768126.129893:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768126/real 1775768126] req@ffff9942099fe300 x1862022294277120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768126.129894:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768126.129900:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768126.129905:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000001:1.0:1775768126.129905:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768126.129907:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000040:0.0:1775768126.129909:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000400:00000010:1.0:1775768126.129910:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768126.129915:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000100:00080000:0.0:1775768126.129916:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000400:00000010:1.0:1775768126.129917:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768126.129919:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768126.129921:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fe300 x1862022294277120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768126.129927:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fe300 x1862022294277120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768126.129930:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768126.129935:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1775768126.129936:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768126.129939:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768126.129940:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:0.0:1775768126.129941:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768126.129942:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1775768126.129946:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768126.129947:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fe300 x1862022294277120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768126.129951:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775768126.129955:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000100:00000001:1.0:1775768126.129956:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1775768126.129960:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768126.129963:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768126.129963:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fe300 x1862022294277120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768151 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1775768126.129968:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fd500. 00000100:00000001:1.0:1775768126.129970:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000040:0.0:1775768126.129971:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00000100:00000001:1.0:1775768126.129973:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1775768126.129974:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1775768126.129975:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:0.0:1775768126.129977:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768126.129977:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768126.129981:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1775768126.129981:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 02000000:00000001:0.0:1775768126.129983:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1775768126.129984:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:0.0:1775768126.129986:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768126.129987:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 02000000:00000010:0.0:1775768126.129992:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220029400. 00000100:00080000:1.0:1775768126.129992:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768126.129996:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:0.0:1775768126.129997:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768126.130000:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fe300 x1862022294277120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768151 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768126.130005:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fd500 x1862022294277504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768126.130012:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294277120:192.168.202.156@tcp:250:kworker.0 00000100:00000001:0.0:1775768126.130016:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775768126.130017:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768126.130019:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768126.130020:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768126.130024:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f246bb2b] to pc [ptlrpcd_rcv+-1] req@ffff9942099fd500 x1862022294277504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775768126.130025:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fe300 x1862022294277120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768151 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768126.130050:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 00000100:00000001:0.0:1775768126.130052:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768126.130052:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768126.130055:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543e400. 00000100:00080000:0.0:1775768126.130056:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1775768126.130057:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768126.130060:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768126.130062:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000100:00080000:0.0:1775768126.130063:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000020:00000001:1.0:1775768126.130064:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1775768126.130066:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 02000000:00000010:1.0:1775768126.130067:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028000. 00000100:00080000:0.0:1775768126.130069:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:1.0:1775768126.130071:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768126.130074:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1775768126.130075:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000010:1.0:1775768126.130075:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fe300. 00000100:00000001:1.0:1775768126.130079:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00080000:0.0:1775768126.130081:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5378 00000100:00000001:1.0:1775768126.130081:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768126.130083:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768126.130089:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:0.0:1775768126.130090:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768126.130092:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768126.130094:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:0.0:1775768126.130096:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:1.0:1775768126.130097:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768126.130099:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768126.130101:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fc700 x1862022294277248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768126.130106:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768126.130108:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768126.130109:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 02000000:00000001:1.0:1775768126.130109:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768126.130111:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768126.130114:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768126.130116:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00100000:1.0:1775768126.130117:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294277248:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768126.130121:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775768126.130122:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768126.130125:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768126.130126:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768126.130127:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000001:1.0:1775768126.130128:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768126.130131:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543e400. 00000100:00000001:0.0:1775768126.130132:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 02000000:00000001:1.0:1775768126.130134:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768126.130136:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768126.130138:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000400:00000200:1.0:1775768126.130139:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:0.0:1775768126.130141:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000010:1.0:1775768126.130142:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000020:00000040:0.0:1775768126.130145:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000400:00000010:1.0:1775768126.130146:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000020:00000001:0.0:1775768126.130150:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000200:1.0:1775768126.130150:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294277248, portal 10 00000020:00000001:0.0:1775768126.130169:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:0.0:1775768126.130173:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768126.130174:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775768126.130175:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000040:0.0:1775768126.130177:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768126.130178:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641198336 : -112966068353280 : ffff9942099fc700) 00000100:00000001:0.0:1775768126.130181:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768126.130183:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00000040:1.0:1775768126.130186:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fc700 x1862022294277248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768151 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768126.130188:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768126.130194:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768126.130196:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:0.0:1775768126.130198:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:1.0:1775768126.130201:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 02000000:00000001:0.0:1775768126.130203:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:1.0:1775768126.130204:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768126.130207:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294277248, offset 0 02000000:00000001:0.0:1775768126.130208:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768126.130212:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:1.0:1775768126.130212:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000008:00100000:0.0:1775768126.130216:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000400:00000200:1.0:1775768126.130216:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00000001:0.0:1775768126.130221:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768126.130221:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00000001:0.0:1775768126.130226:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768126.130226:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768126.130230:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000100:00000010:0.0:1775768126.130232:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ff800. 00000400:00000200:1.0:1775768126.130232:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768126.130235:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000020:00000040:0.0:1775768126.130237:0:13471:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00000400:00000200:1.0:1775768126.130239:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225339:225339:256:4294967295] 192.168.202.156@tcp LPNI seq info [225335:225335:8:4294967295] 02000000:00000001:0.0:1775768126.130242:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000200:1.0:1775768126.130243:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:0.0:1775768126.130245:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768126.130249:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768126.130252:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:1.0:1775768126.130252:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 02000000:00000001:0.0:1775768126.130255:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775768126.130259:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 02000000:00000010:0.0:1775768126.130260:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220029800. 00000800:00000010:1.0:1775768126.130262:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000100:00000001:0.0:1775768126.130264:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775768126.130268:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768126.130270:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000100:00100000:0.0:1775768126.130271:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ff800 x1862022294277632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.1:1775768126.130279:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:0.0:1775768126.130284:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000200:1.1:1775768126.130284:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000100:00000001:0.0:1775768126.130287:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1775768126.130290:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768126.130293:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000100:00000040:0.0:1775768126.130294:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000aa4e8b71] to pc [ptlrpcd_rcv+-1] req@ffff9942099ff800 x1862022294277632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775768126.130296:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768126.130303:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768126.130306:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000100:00000001:0.0:1775768126.130307:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775768126.130308:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768126.130311:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:0.0:1775768126.130312:0:13471:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5388) 00000100:00000200:1.0:1775768126.130315:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fc700 x1862022294277248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768151 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768126.130321:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768126.130324:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fc700 x1862022294277248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768151 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768126.130328:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768126.130329:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768126.130331:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768126.130333:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:1.0:1775768126.130335:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768126.130337:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300050963 00000100:00000001:1.0:1775768126.130339:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300052285 00000100:00000001:1.0:1775768126.130340:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300050963 00000100:00000001:1.0:1775768126.130341:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300052285 00000100:00000001:1.0:1775768126.130342:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768126.130344:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768126.130345:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768126.130347:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768126/real 1775768126] req@ffff9942099fc700 x1862022294277248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768126.130351:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768126.130352:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768126.130352:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768126.130354:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768126.130354:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768126.130355:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768126.130357:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fc700 x1862022294277248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768126.130361:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fc700 x1862022294277248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768126.130365:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768126.130366:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768126.130367:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768126.130367:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768126.130369:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fc700 x1862022294277248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768126.130373:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768126.130374:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fc700 x1862022294277248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768151 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768126.130379:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768126.130380:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768126.130381:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768126.130382:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768126.130383:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768126.130384:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768126.130386:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768126.130387:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768126.130388:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768126.130390:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fc700 x1862022294277248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768151 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768126.130394:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294277248:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768126.130397:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768126.130398:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ff100 x1862022294277376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768126.130402:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768126.130403:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768126.130404:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768126.130405:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768126.130406:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768126.130408:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294277376:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768126.130410:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768126.130411:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768126.130412:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768126.130413:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768126.130414:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543d400. 02000000:00000001:1.0:1775768126.130416:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768126.130418:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768126.130419:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000400:00000010:1.0:1775768126.130421:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000200:1.0:1775768126.130422:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294277376, portal 10 00000100:00000001:1.0:1775768126.130423:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768126.130424:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641209088 : -112966068342528 : ffff9942099ff100) 00000100:00000040:1.0:1775768126.130427:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ff100 x1862022294277376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768151 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768126.130431:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768126.130432:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768126.130433:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768126.130435:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294277376, offset 0 00000400:00000200:1.0:1775768126.130437:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768126.130438:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768126.130440:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768126.130442:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768126.130443:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768126.130444:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768126.130446:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768126.130447:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225340:225340:256:4294967295] 192.168.202.156@tcp LPNI seq info [225336:225336:8:4294967295] 00000400:00000200:1.0:1775768126.130450:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768126.130454:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768126.130456:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768126.130458:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000800:00000200:1.0:1775768126.130460:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768126.130461:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775768126.130464:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768126.130465:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000100:1.0:1775768126.130467:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768126.130468:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768126.130469:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768126.130471:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768126.130473:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768126.130474:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768126.130475:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768126.130477:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ff100 x1862022294277376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768151 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768126.130481:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768126.130483:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ff100 x1862022294277376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768151 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768126.130486:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768126.130487:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768126.130488:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768126.130490:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:1.0:1775768126.130492:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768126.130493:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300050955 00000100:00000001:1.0:1775768126.130494:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300052285 00000100:00000001:1.0:1775768126.130495:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300050955 00000100:00000001:1.0:1775768126.130497:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300052285 00000100:00000001:1.0:1775768126.130497:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768126.130498:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768126.130499:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768126.130501:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768126/real 1775768126] req@ffff9942099ff100 x1862022294277376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768126.130505:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768126.130506:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768126.130506:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768126.130507:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768126.130508:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768126.130509:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768126.130511:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ff100 x1862022294277376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768126.130515:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ff100 x1862022294277376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768126.130520:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768126.130521:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768126.130521:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768126.130522:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768126.130524:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ff100 x1862022294277376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768126.130527:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768126.130529:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ff100 x1862022294277376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768151 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768126.130533:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768126.130534:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768126.130534:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768126.130535:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768126.130536:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768126.130537:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768126.130539:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768126.130540:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768126.130541:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768126.130543:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ff100 x1862022294277376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768151 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768126.130547:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294277376:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768126.130549:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768126.130550:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fd500 x1862022294277504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768126.130554:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768126.130555:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768126.130556:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768126.130557:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768126.130559:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768126.130560:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294277504:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768126.130562:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768126.130563:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768126.130564:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768126.130564:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768126.130566:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543d000. 02000000:00000001:1.0:1775768126.130567:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768126.130569:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768126.130570:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000400:00000010:1.0:1775768126.130571:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000200:1.0:1775768126.130573:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294277504, portal 4 00000100:00000001:1.0:1775768126.130574:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768126.130575:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641201920 : -112966068349696 : ffff9942099fd500) 00000100:00000040:1.0:1775768126.130577:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fd500 x1862022294277504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768151 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768126.130581:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768126.130582:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768126.130583:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768126.130584:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294277504, offset 0 00000400:00000200:1.0:1775768126.130586:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768126.130588:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768126.130589:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768126.130590:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768126.130592:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768126.130593:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768126.130594:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768126.130596:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225341:225341:256:4294967295] 192.168.202.156@tcp LPNI seq info [225337:225337:8:4294967295] 00000400:00000200:1.0:1775768126.130599:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768126.130602:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768126.130605:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768126.130606:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000800:00000200:1.0:1775768126.130608:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768126.130610:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775768126.130612:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768126.130613:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000100:1.0:1775768126.130615:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768126.130616:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768126.130618:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768126.130619:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768126.130621:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768126.130622:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768126.130623:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768126.130625:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fd500 x1862022294277504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768151 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768126.130629:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768126.130630:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fd500 x1862022294277504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768151 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768126.130633:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768126.130634:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768126.130636:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768126.130637:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:1.0:1775768126.130639:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768126.130640:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300050978 00000100:00000001:1.0:1775768126.130642:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300052285 00000100:00000001:1.0:1775768126.130643:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300050978 00000100:00000001:1.0:1775768126.130644:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300052285 00000100:00000001:1.0:1775768126.130645:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768126.130646:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768126.130647:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768126.130648:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768126/real 1775768126] req@ffff9942099fd500 x1862022294277504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768126.130653:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768126.130654:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768126.130655:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768126.130656:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768126.130657:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768126.130657:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768126.130659:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fd500 x1862022294277504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768126.130663:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fd500 x1862022294277504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768126.130666:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768126.130667:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768126.130668:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768126.130668:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768126.130670:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fd500 x1862022294277504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768126.130686:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768126.130689:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fd500 x1862022294277504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768151 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768126.130693:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768126.130695:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768126.130696:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768126.130697:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768126.130699:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768126.130701:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768126.130703:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768126.130706:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768126.130708:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768126.130712:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fd500 x1862022294277504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768151 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768126.130720:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294277504:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768126.130723:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768126.130725:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768126.130727:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fc700 x1862022294277248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768151 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768126.130734:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768126.130736:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768126.130737:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543e400. 02000000:00000001:1.0:1775768126.130740:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768126.130741:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768126.130742:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:1.0:1775768126.130744:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768126.130746:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002b800. 02000000:00000001:1.0:1775768126.130748:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768126.130749:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768126.130749:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fc700. 00000100:00000001:1.0:1775768126.130751:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768126.130752:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768126.130753:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768126.130755:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ff100 x1862022294277376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768151 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768126.130759:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768126.130759:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768126.130760:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543d400. 02000000:00000001:1.0:1775768126.130762:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768126.130762:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768126.130763:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:1.0:1775768126.130764:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768126.130765:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002bc00. 02000000:00000001:1.0:1775768126.130766:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768126.130767:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768126.130768:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ff100. 00000100:00000001:1.0:1775768126.130769:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768126.130770:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768126.130771:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768126.130772:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fd500 x1862022294277504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768151 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768126.130776:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768126.130776:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768126.130777:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543d000. 02000000:00000001:1.0:1775768126.130778:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768126.130778:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768126.130779:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:1.0:1775768126.130780:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768126.130781:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029400. 02000000:00000001:1.0:1775768126.130782:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768126.130782:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768126.130783:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fd500. 00000100:00000001:1.0:1775768126.130784:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768126.130784:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768126.130785:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768126.130787:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768126.130788:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768126.130789:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768126.130790:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768126.130791:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768126.130793:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ff800 x1862022294277632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768126.130796:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768126.130797:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768126.130798:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768126.130799:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768126.130800:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768126.130801:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294277632:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768126.130803:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768126.130805:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768126.130806:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768126.130807:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768126.130809:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543d000. 02000000:00000001:1.0:1775768126.130810:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768126.130812:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768126.130814:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000400:00000010:1.0:1775768126.130815:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000200:1.0:1775768126.130817:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294277632, portal 4 00000100:00000001:1.0:1775768126.130819:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768126.130819:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641210880 : -112966068340736 : ffff9942099ff800) 00000100:00000040:1.0:1775768126.130823:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ff800 x1862022294277632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768151 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768126.130830:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768126.130832:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768126.130834:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768126.130836:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294277632, offset 0 00000400:00000200:1.0:1775768126.130839:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768126.130843:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768126.130846:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768126.130848:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768126.130851:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768126.130853:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768126.130855:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768126.130857:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225342:225342:256:4294967295] 192.168.202.156@tcp LPNI seq info [225338:225338:8:4294967295] 00000400:00000200:1.0:1775768126.130861:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768126.130866:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768126.130870:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768126.130874:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000800:00000200:1.0:1775768126.130877:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768126.130879:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775768126.130882:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768126.130884:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000100:1.0:1775768126.130887:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768126.130890:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768126.130892:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768126.130895:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768126.130897:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768126.130899:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768126.130901:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768126.130904:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ff800 x1862022294277632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768151 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768126.130911:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768126.130913:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ff800 x1862022294277632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768151 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768126.130919:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768126.130921:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768126.130924:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768126.130926:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:1.0:1775768126.130928:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768126.130929:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300050981 00000100:00000001:1.0:1775768126.130931:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300052285 00000100:00000001:1.0:1775768126.130932:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300050981 00000100:00000001:1.0:1775768126.130933:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300052285 00000100:00000001:1.0:1775768126.130935:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768126.130936:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768126.130937:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768126.130939:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768126/real 1775768126] req@ffff9942099ff800 x1862022294277632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768126.130943:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768126.130944:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768126.130944:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768126.130946:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768126.130947:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768126.130948:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768126.130950:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ff800 x1862022294277632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768126.130954:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ff800 x1862022294277632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768126.130957:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768126.130958:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768126.130959:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768126.130960:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768126.130962:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ff800 x1862022294277632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768126.130965:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768126.130967:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ff800 x1862022294277632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768151 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768126.130970:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768126.130972:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768126.130973:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768126.130973:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768126.130974:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768126.130977:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768126.130978:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768126.130980:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768126.130981:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768126.130983:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ff800 x1862022294277632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768151 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768126.130986:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294277632:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768126.130989:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768126.130990:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768126.130992:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ff800 x1862022294277632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768151 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768126.130996:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768126.130997:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768126.130998:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543d000. 02000000:00000001:1.0:1775768126.130999:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768126.131000:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768126.131000:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:1.0:1775768126.131001:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768126.131002:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029800. 02000000:00000001:1.0:1775768126.131004:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768126.131005:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768126.131005:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ff800. 00000100:00000001:1.0:1775768126.131007:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768126.131008:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768126.131009:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768126.131011:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768126.131011:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768126.131012:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768126.131013:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768126.193253:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768126.193257:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768126.193261:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768126.193262:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768126.193264:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768126.193265:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768126.193267:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768126.193268:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768126.193270:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768126.193271:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768126.193324:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768126.193328:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768126.193332:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768126.257273:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768126.257279:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768126.257280:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768126.257284:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768126.257287:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768126.257290:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768126.257292:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768126.257294:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768126.257295:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768127.153253:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768127.153253:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768127.153258:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768127.153259:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768127.153261:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768127.153263:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768127.153266:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775768127.153267:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775768127.153270:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768127.153272:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768127.153274:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768127.153275:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768127.153277:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768127.153279:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768127.153281:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768127.153286:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768127.153288:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768127.153290:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768127.153355:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768127.153358:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768127.153359:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768127.153361:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768127.153363:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768127.153365:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768127.153366:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768127.153368:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768127.153369:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768127.217252:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768127.217256:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768127.217260:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768127.217263:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768127.217265:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768127.217268:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768127.217270:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768127.217272:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768127.217274:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768127.217276:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768127.217278:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768127.217279:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768127.217280:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768127.217280:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768127.217281:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768127.217282:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768127.217283:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768127.217284:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768127.217285:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768127.217286:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768127.217287:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768127.217287:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768127.217288:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768127.217289:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768127.217290:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768127.217291:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768127.217292:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768127.217293:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768127.217294:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768127.217295:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768127.217295:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768127.217296:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768127.217297:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768127.217298:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768127.217345:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768127.217348:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768127.217350:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768127.281262:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768127.281268:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768127.281270:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768127.281275:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768127.281279:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768127.281283:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768127.281285:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768127.281288:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768127.281291:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768127.473261:0:14953:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4300052285, at 4300052288 00000100:00000040:1.0:1775768127.473262:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4300052285, at 4300052288 00000100:00000010:0.0:1775768127.473277:0:14953:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cba1e0. 00000100:00000010:1.0:1775768127.473282:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cbad80. 00000100:00000001:1.0:1775768128.177225:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768128.177227:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768128.177232:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768128.177232:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768128.177237:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768128.177237:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768128.177243:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775768128.177243:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768128.177248:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768128.177249:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768128.177253:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768128.177254:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768128.177255:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768128.177259:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768128.177261:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768128.177262:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768128.177264:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768128.177267:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768128.177268:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775768128.177268:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768128.177272:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768128.177273:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768128.177276:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768128.177277:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768128.177281:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768128.177284:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768128.177286:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768128.241248:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768128.241253:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768128.241259:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768128.241260:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768128.241264:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768128.241265:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768128.241268:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768128.241269:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768128.241272:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768128.241273:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768128.241323:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768128.241328:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768128.241331:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768128.305252:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768128.305256:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768128.305257:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768128.305261:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768128.305264:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768128.305267:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768128.305268:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768128.305276:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768128.305278:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768129.201253:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768129.201253:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768129.201260:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768129.201260:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768129.201263:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768129.201263:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768129.201266:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:0.0:1775768129.201269:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:1.0:1775768129.201269:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768129.201271:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768129.201274:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768129.201274:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768129.201275:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768129.201278:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768129.201280:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768129.201280:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768129.201281:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768129.201284:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768129.201286:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768129.201286:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768129.201287:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768129.201289:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768129.201290:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768129.201294:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768129.201295:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768129.201298:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768129.201301:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768129.265228:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768129.265232:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768129.265237:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768129.265238:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768129.265240:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768129.265241:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768129.265243:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768129.265244:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768129.265246:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768129.265246:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768129.265282:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768129.265285:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768129.265287:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768129.329207:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768129.329212:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768129.329213:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768129.329216:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768129.329219:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768129.329222:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768129.329223:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768129.329225:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768129.329226:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768130.225217:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768130.225219:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768130.225220:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768130.225223:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768130.225224:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768130.225224:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768130.225227:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768130.225227:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768130.225227:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768130.225234:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:1.0:1775768130.225234:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775768130.225234:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768130.225239:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768130.225240:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768130.225240:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768130.225243:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768130.225244:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768130.225244:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768130.225249:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768130.225257:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768130.225259:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768130.225260:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768130.225262:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768130.225265:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768130.225265:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768130.225267:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768130.225270:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768130.289241:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768130.289244:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768130.289248:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768130.289254:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768130.289257:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768130.289258:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768130.289259:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768130.289260:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768130.289262:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768130.289263:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768130.289309:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768130.289312:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768130.289314:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768130.353252:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768130.353256:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768130.353257:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768130.353261:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768130.353264:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768130.353267:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768130.353268:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768130.353270:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768130.353272:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768131.249248:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768131.249253:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768131.249254:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768131.249258:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768131.249259:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768131.249262:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768131.249262:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:0.0:1775768131.249267:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775768131.249267:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768131.249270:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768131.249271:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768131.249273:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768131.249275:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1775768131.249276:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775768131.249276:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768131.249280:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768131.249280:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768131.249284:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1775768131.249286:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775768131.249286:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768131.249287:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768131.249290:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1775768131.249292:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:1.0:1775768131.249295:0:36:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775768131.249295:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768131.249300:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768131.249301:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:3.0:1775768131.249304:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:1.0:1775768131.249309:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5383 00000100:00000001:3.0:1775768131.249314:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768131.249318:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000400:00000200:1.0:1775768131.249321:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775768131.249321:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768131.249326:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768131.249331:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775768131.249335:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775768131.249340:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768131.249344:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768131.249347:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768131.249350:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775768131.249354:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775768131.249357:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:1.0:1775768131.249360:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775768131.249362:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:1.0:1775768131.249365:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:1.0:1775768131.249368:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:1.0:1775768131.249371:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768131.249373:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768131.249375:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768131.249377:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775768131.249380:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:1.0:1775768131.249383:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775768131.249385:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768131.249388:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775768131.249391:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768131.249394:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768131.249396:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775768131.249399:0:36:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775768131.249407:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207888e00. 00000020:00000040:1.0:1775768131.249411:0:36:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:1.0:1775768131.249414:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768131.249416:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768131.249419:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775768131.249421:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775768131.249425:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775768131.249433:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543e400. 00000100:00000001:1.0:1775768131.249439:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768131.249448:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207888e00 x1862022294277760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775768131.249457:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775768131.249462:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768131.249466:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001812f966] to pc [ptlrpcd_rcv+-1] req@ffff994207888e00 x1862022294277760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768131.249485:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768131.249488:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768131.249491:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775768131.249493:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775768131.249495:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768131.249498:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775768131.249501:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5383 00000100:00000001:2.0:1775768131.249504:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:1.0:1775768131.249507:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768131.249507:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768131.249510:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000400:00000200:1.0:1775768131.249513:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775768131.249516:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775768131.249519:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768131.249522:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768131.249523:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207888e00 x1862022294277760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775768131.249526:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768131.249531:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775768131.249537:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:2.0:1775768131.249538:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000040:1.0:1775768131.249539:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000100:00000001:2.0:1775768131.249540:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768131.249542:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000001:2.0:1775768131.249544:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:1.0:1775768131.249545:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 02000000:00000001:2.0:1775768131.249546:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1775768131.249550:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 02000000:00000001:2.0:1775768131.249551:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768131.249556:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00100000:2.0:1775768131.249556:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207888e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294277760:192.168.202.156@tcp:250:kworker.0 00000100:00000040:1.0:1775768131.249563:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768131.249563:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1775768131.249567:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1775768131.249568:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:1.0:1775768131.249570:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000001:2.0:1775768131.249572:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768131.249573:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775768131.249577:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 02000000:00000001:2.0:1775768131.249577:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768131.249584:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f800. 02000000:00000001:2.0:1775768131.249586:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768131.249591:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775768131.249594:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768131.249595:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:1.0:1775768131.249598:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:2.0:1775768131.249601:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 02000000:00000001:1.0:1775768131.249604:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768131.249607:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768131.249609:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000010:2.0:1775768131.249609:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000008:00100000:1.0:1775768131.249614:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775768131.249618:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768131.249619:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775768131.249623:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207889880. 00000100:00000200:2.0:1775768131.249637:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294277760, portal 25 00000020:00000040:1.0:1775768131.249640:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000100:00000001:2.0:1775768131.249641:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768131.249645:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606121984 : -112966103429632 : ffff994207888e00) 02000000:00000001:1.0:1775768131.249646:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768131.249648:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768131.249651:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775768131.249653:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775768131.249655:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768131.249655:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207888e00 x1862022294277760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768156 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1775768131.249661:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543d400. 00000100:00000001:1.0:1775768131.249667:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768131.249671:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:1.0:1775768131.249675:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207889880 x1862022294277888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:2.0:1775768131.249678:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768131.249683:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000001:1.0:1775768131.249688:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000200:2.0:1775768131.249688:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294277760, offset 0 00000100:00000001:1.0:1775768131.249692:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768131.249697:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00000040:1.0:1775768131.249698:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f11856b8] to pc [ptlrpcd_rcv+-1] req@ffff994207889880 x1862022294277888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1775768131.249705:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768131.249712:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768131.249713:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00080000:1.0:1775768131.249716:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:2.0:1775768131.249719:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:1.0:1775768131.249723:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000200:2.0:1775768131.249725:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000100:00000001:1.0:1775768131.249726:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775768131.249728:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:2.0:1775768131.249729:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000100:00000001:1.0:1775768131.249733:0:36:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:2.0:1775768131.249733:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00080000:1.0:1775768131.249738:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5383 00000400:00000200:2.0:1775768131.249739:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225343:225343:256:4294967295] 192.168.202.156@tcp LPNI seq info [225339:225339:8:4294967295] 00000400:00000200:1.0:1775768131.249746:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768131.249746:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768131.249753:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775768131.249755:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775768131.249762:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000200:2.0:1775768131.249763:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000100:00000001:1.0:1775768131.249767:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768131.249772:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000800:00000200:2.0:1775768131.249772:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00000001:1.0:1775768131.249776:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000800:00000010:2.0:1775768131.249777:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000020:00000001:1.0:1775768131.249779:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775768131.249782:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:1.0:1775768131.249785:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000800:00000200:2.0:1775768131.249786:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000040:1.0:1775768131.249788:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000800:00000200:2.0:1775768131.249789:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000020:00000001:1.0:1775768131.249792:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:1.0:1775768131.249797:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000800:00000200:2.1:1775768131.249797:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768131.249803:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000100:00000040:1.0:1775768131.249804:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768131.249810:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768131.249814:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000800:00000100:2.0:1775768131.249814:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000100:00000001:1.0:1775768131.249817:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000800:00000010:2.0:1775768131.249819:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000020:00000040:1.0:1775768131.249822:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:1.0:1775768131.249826:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000400:00000200:2.0:1775768131.249827:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:1.0:1775768131.249831:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768131.249835:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:2.0:1775768131.249835:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 02000000:00000001:1.0:1775768131.249840:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1775768131.249842:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 02000000:00000001:1.0:1775768131.249845:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775768131.249846:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:1.0:1775768131.249850:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775768131.249853:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000008:00100000:1.0:1775768131.249854:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775768131.249858:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1775768131.249858:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207888e00 x1862022294277760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768156 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768131.249864:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775768131.249869:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207888700. 00000020:00000040:1.0:1775768131.249871:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000100:00000001:2.0:1775768131.249871:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 02000000:00000001:1.0:1775768131.249875:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000040:2.0:1775768131.249875:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207888e00 x1862022294277760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768156 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1775768131.249877:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768131.249880:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775768131.249882:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775768131.249885:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768131.249886:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775768131.249888:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543d000. 00000100:00000001:2.0:1775768131.249889:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768131.249893:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768131.249897:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768131.249899:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207888700 x1862022294278016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1775768131.249905:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768131.249910:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775768131.249910:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4300057405 00000100:00000001:1.0:1775768131.249911:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768131.249917:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009f2ba301] to pc [ptlrpcd_rcv+-1] req@ffff994207888700 x1862022294278016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1775768131.249919:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d6a20. 00000100:00000001:1.0:1775768131.249925:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768131.249927:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768131.249929:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1775768131.249929:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775768131.249931:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775768131.249934:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768131.249936:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775768131.249937:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4300057405 00000100:00080000:1.0:1775768131.249938:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5383 00000400:00000200:1.0:1775768131.249943:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000010:2.0:1775768131.249943:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d6ea0. 00000400:00000200:1.0:1775768131.249948:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775768131.249949:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775768131.249951:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768131.249953:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768131.249955:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768131.249957:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775768131.249958:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775768131.249962:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:2.0:1775768131.249963:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000040:1.0:1775768131.249966:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000100:00000001:2.0:1775768131.249966:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768131.249970:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:2.0:1775768131.249970:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000020:00000040:1.0:1775768131.249974:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000100:00100000:2.0:1775768131.249977:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768131/real 1775768131] req@ffff994207888e00 x1862022294277760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1775768131.249978:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:1.0:1775768131.249984:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:1.0:1775768131.249991:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000001:2.0:1775768131.249991:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768131.249994:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1775768131.249995:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:2.0:1775768131.249996:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000100:00000040:1.0:1775768131.249999:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000200:2.0:1775768131.250000:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768131.250003:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:1.0:1775768131.250004:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:2.0:1775768131.250006:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000040:1.0:1775768131.250009:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00000200:2.0:1775768131.250011:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207888e00 x1862022294277760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768131.250015:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775768131.250021:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1775768131.250023:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207888e00 x1862022294277760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1775768131.250024:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775768131.250029:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768131.250029:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768131.250032:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1775768131.250034:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768131.250035:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768131.250037:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768131.250038:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00100000:2.0:1775768131.250041:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207888e00 x1862022294277760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1775768131.250042:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:1.0:1775768131.250048:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768131.250050:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775768131.250054:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff99420788b100. 00000100:00000001:2.0:1775768131.250057:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:1.0:1775768131.250059:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00000100:00000040:2.0:1775768131.250063:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207888e00 x1862022294277760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768156 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1775768131.250064:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768131.250066:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768131.250070:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775768131.250072:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775768131.250075:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768131.250077:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000010:1.0:1775768131.250080:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543f000. 00000100:00000001:2.0:1775768131.250082:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768131.250085:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768131.250085:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768131.250089:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768131.250092:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00100000:1.0:1775768131.250093:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff99420788b100 x1862022294278144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775768131.250095:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768131.250103:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:1.0:1775768131.250105:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775768131.250109:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768131.250110:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768131.250114:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768131.250116:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e26cf977] to pc [ptlrpcd_rcv+-1] req@ffff99420788b100 x1862022294278144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775768131.250123:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207888e00 x1862022294277760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768156 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768131.250127:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768131.250131:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768131.250138:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775768131.250141:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775768131.250143:0:36:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775768131.250144:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207888e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294277760:192.168.202.156@tcp:250:kworker.0 00000100:00000001:1.0:1775768131.250148:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775768131.250153:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768131.250172:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768131.250175:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207888e00 x1862022294277760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768156 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768131.250179:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 00000100:00080000:1.0:1775768131.250180:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5383 02000000:00000001:2.0:1775768131.250182:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768131.250185:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f800. 00000400:00000200:1.0:1775768131.250189:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:2.0:1775768131.250189:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768131.250192:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768131.250193:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000400:00000200:1.0:1775768131.250196:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000020:00000001:2.0:1775768131.250196:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00080000:1.0:1775768131.250200:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 02000000:00000010:2.0:1775768131.250200:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543e400. 02000000:00000001:2.0:1775768131.250204:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000040:1.0:1775768131.250206:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 02000000:00000001:2.0:1775768131.250207:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768131.250209:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207888e00. 00000100:00000001:1.0:1775768131.250210:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775768131.250212:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000040:1.0:1775768131.250215:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768131.250215:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768131.250218:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768131.250220:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775768131.250223:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775768131.250223:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000100:00000001:2.0:1775768131.250224:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:1.0:1775768131.250225:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:2.0:1775768131.250226:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1775768131.250227:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000100:00000001:2.0:1775768131.250228:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1775768131.250231:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000001:2.0:1775768131.250232:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768131.250234:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000001:1.0:1775768131.250235:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:2.0:1775768131.250239:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207889880 x1862022294277888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775768131.250240:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768131.250243:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768131.250246:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768131.250250:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775768131.250255:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00000001:2.0:1775768131.250256:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:1.0:1775768131.250259:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775768131.250259:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768131.250262:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1775768131.250263:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768131.250264:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768131.250269:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775768131.250269:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768131.250274:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:2.0:1775768131.250274:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207889880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294277888:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768131.250277:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768131.250279:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768131.250280:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1775768131.250283:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:2.0:1775768131.250283:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768131.250286:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:1.0:1775768131.250287:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 02000000:00000010:2.0:1775768131.250289:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f800. 00000008:00000001:1.0:1775768131.250291:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768131.250293:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768131.250295:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768131.250300:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000010:1.0:1775768131.250301:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207889f80. 00000020:00000040:1.0:1775768131.250304:0:36:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00000400:00000010:2.0:1775768131.250305:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 02000000:00000001:1.0:1775768131.250308:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768131.250310:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775768131.250311:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000001:1.0:1775768131.250314:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775768131.250316:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000200:2.0:1775768131.250316:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294277888, portal 10 02000000:00000001:1.0:1775768131.250319:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768131.250320:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768131.250323:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606124672 : -112966103426944 : ffff994207889880) 02000000:00000010:1.0:1775768131.250326:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543c400. 00000100:00000040:2.0:1775768131.250331:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207889880 x1862022294277888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768156 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768131.250332:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768131.250338:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207889f80 x1862022294278272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775768131.250346:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768131.250351:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000100:00000001:1.0:1775768131.250352:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775768131.250354:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775768131.250354:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768131.250357:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294277888, offset 0 00000100:00000040:1.0:1775768131.250358:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ed2274f4] to pc [ptlrpcd_rcv+-1] req@ffff994207889f80 x1862022294278272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1775768131.250365:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00000001:1.0:1775768131.250371:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768131.250372:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000040:1.0:1775768131.250376:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5393) 00000400:00000200:2.0:1775768131.250376:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768131.250381:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768131.250383:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768131.250385:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768131.250387:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768131.250390:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225344:225344:256:4294967295] 192.168.202.156@tcp LPNI seq info [225340:225340:8:4294967295] 00000400:00000200:2.0:1775768131.250393:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768131.250397:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768131.250400:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768131.250402:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768131.250404:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768131.250407:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1775768131.250409:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768131.250411:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000100:2.0:1775768131.250413:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768131.250414:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768131.250416:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768131.250418:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768131.250420:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768131.250421:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768131.250423:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768131.250425:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207889880 x1862022294277888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768156 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768131.250430:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768131.250432:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207889880 x1862022294277888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768156 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768131.250436:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768131.250437:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768131.250439:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768131.250441:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:2.0:1775768131.250443:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768131.250444:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300056083 00000100:00000001:2.0:1775768131.250447:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300057405 00000100:00000001:2.0:1775768131.250448:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300056083 00000100:00000001:2.0:1775768131.250450:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300057405 00000100:00000001:2.0:1775768131.250451:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768131.250453:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768131.250454:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768131.250456:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768131/real 1775768131] req@ffff994207889880 x1862022294277888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768131.250461:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768131.250462:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768131.250462:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768131.250464:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768131.250465:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768131.250466:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768131.250468:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207889880 x1862022294277888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768131.250473:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207889880 x1862022294277888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768131.250476:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768131.250477:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768131.250478:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768131.250479:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768131.250480:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207889880 x1862022294277888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768131.250484:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768131.250486:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207889880 x1862022294277888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768156 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768131.250489:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768131.250490:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768131.250491:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768131.250492:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768131.250493:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768131.250494:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768131.250496:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768131.250498:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768131.250499:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768131.250502:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207889880 x1862022294277888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768156 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768131.250506:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207889880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294277888:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768131.250508:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768131.250509:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207888700 x1862022294278016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768131.250513:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768131.250514:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768131.250515:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768131.250516:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768131.250517:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768131.250519:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207888700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294278016:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768131.250521:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768131.250522:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768131.250523:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768131.250524:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768131.250526:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f000. 02000000:00000001:2.0:1775768131.250527:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768131.250529:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768131.250530:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768131.250532:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768131.250534:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294278016, portal 10 00000100:00000001:2.0:1775768131.250535:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768131.250536:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606120192 : -112966103431424 : ffff994207888700) 00000100:00000040:2.0:1775768131.250539:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207888700 x1862022294278016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768156 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768131.250544:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768131.250546:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768131.250547:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768131.250548:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294278016, offset 0 00000400:00000200:2.0:1775768131.250550:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768131.250553:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768131.250556:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768131.250559:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768131.250561:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768131.250563:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768131.250565:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768131.250568:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225345:225345:256:4294967295] 192.168.202.156@tcp LPNI seq info [225341:225341:8:4294967295] 00000400:00000200:2.0:1775768131.250572:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768131.250578:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768131.250582:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768131.250585:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768131.250589:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768131.250592:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1775768131.250596:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768131.250598:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000100:2.0:1775768131.250601:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768131.250604:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768131.250606:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768131.250610:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768131.250613:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768131.250614:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768131.250617:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768131.250620:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207888700 x1862022294278016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768156 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768131.250641:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768131.250643:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207888700 x1862022294278016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768156 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768131.250648:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768131.250649:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768131.250651:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768131.250653:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:2.0:1775768131.250656:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768131.250657:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300056075 00000100:00000001:2.0:1775768131.250659:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300057405 00000100:00000001:2.0:1775768131.250661:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300056075 00000100:00000001:2.0:1775768131.250662:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300057405 00000100:00000001:2.0:1775768131.250663:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768131.250665:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768131.250666:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768131.250668:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768131/real 1775768131] req@ffff994207888700 x1862022294278016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768131.250672:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768131.250672:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768131.250673:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768131.250674:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768131.250675:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768131.250676:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768131.250678:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207888700 x1862022294278016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768131.250682:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207888700 x1862022294278016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768131.250686:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768131.250687:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768131.250688:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768131.250689:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768131.250691:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207888700 x1862022294278016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768131.250694:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768131.250696:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207888700 x1862022294278016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768156 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768131.250701:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768131.250702:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768131.250704:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768131.250705:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768131.250706:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768131.250709:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768131.250711:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768131.250714:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768131.250716:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768131.250720:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207888700 x1862022294278016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768156 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768131.250728:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207888700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294278016:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768131.250732:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768131.250735:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff99420788b100 x1862022294278144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768131.250741:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768131.250743:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768131.250745:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768131.250747:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768131.250749:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768131.250751:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff99420788b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294278144:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768131.250754:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768131.250756:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768131.250758:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768131.250759:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768131.250763:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1e400. 02000000:00000001:2.0:1775768131.250765:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768131.250769:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768131.250771:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768131.250774:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768131.250777:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294278144, portal 4 00000100:00000001:2.0:1775768131.250779:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768131.250781:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606130944 : -112966103420672 : ffff99420788b100) 00000100:00000040:2.0:1775768131.250785:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff99420788b100 x1862022294278144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768156 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768131.250792:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768131.250794:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768131.250796:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768131.250798:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294278144, offset 0 00000400:00000200:2.0:1775768131.250802:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768131.250806:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768131.250809:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768131.250811:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768131.250812:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768131.250815:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768131.250817:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768131.250820:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225346:225346:256:4294967295] 192.168.202.156@tcp LPNI seq info [225342:225342:8:4294967295] 00000400:00000200:2.0:1775768131.250824:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768131.250830:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768131.250834:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768131.250838:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768131.250841:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768131.250844:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1775768131.250848:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768131.250850:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000100:2.0:1775768131.250854:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768131.250856:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768131.250858:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768131.250862:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768131.250864:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768131.250867:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768131.250869:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768131.250871:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff99420788b100 x1862022294278144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768156 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768131.250876:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768131.250878:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff99420788b100 x1862022294278144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768156 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768131.250882:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768131.250883:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768131.250885:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768131.250888:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:2.0:1775768131.250891:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768131.250893:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300056098 00000100:00000001:2.0:1775768131.250895:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300057405 00000100:00000001:2.0:1775768131.250897:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300056098 00000100:00000001:2.0:1775768131.250899:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300057405 00000100:00000001:2.0:1775768131.250902:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768131.250903:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768131.250905:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768131.250909:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768131/real 1775768131] req@ffff99420788b100 x1862022294278144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768131.250917:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768131.250918:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768131.250919:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768131.250922:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768131.250923:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768131.250925:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768131.250928:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff99420788b100 x1862022294278144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768131.250935:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff99420788b100 x1862022294278144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768131.250941:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768131.250942:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768131.250944:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768131.250945:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768131.250948:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff99420788b100 x1862022294278144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768131.250953:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768131.250956:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff99420788b100 x1862022294278144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768156 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768131.250961:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768131.250964:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768131.250965:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768131.250966:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768131.250968:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768131.250971:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768131.250973:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768131.250976:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768131.250979:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768131.250982:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff99420788b100 x1862022294278144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768156 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768131.250990:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff99420788b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294278144:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768131.250992:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768131.250994:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768131.250995:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207889880 x1862022294277888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768156 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768131.250999:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768131.251000:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768131.251002:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f800. 02000000:00000001:2.0:1775768131.251004:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768131.251004:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768131.251006:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:2.0:1775768131.251007:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768131.251008:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543d400. 02000000:00000001:2.0:1775768131.251010:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768131.251011:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768131.251011:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207889880. 00000100:00000001:2.0:1775768131.251013:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768131.251014:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768131.251015:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768131.251017:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207888700 x1862022294278016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768156 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768131.251021:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768131.251021:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768131.251022:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f000. 02000000:00000001:2.0:1775768131.251023:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768131.251024:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768131.251024:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:2.0:1775768131.251025:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768131.251026:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543d000. 02000000:00000001:2.0:1775768131.251027:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768131.251028:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768131.251028:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207888700. 00000100:00000001:2.0:1775768131.251029:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768131.251030:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768131.251031:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768131.251032:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff99420788b100 x1862022294278144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768156 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768131.251035:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768131.251036:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768131.251036:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1e400. 02000000:00000001:2.0:1775768131.251038:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768131.251038:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768131.251039:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:2.0:1775768131.251041:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768131.251042:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543f000. 02000000:00000001:2.0:1775768131.251043:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768131.251044:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768131.251044:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff99420788b100. 00000100:00000001:2.0:1775768131.251046:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768131.251046:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768131.251047:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768131.251049:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768131.251050:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768131.251051:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768131.251052:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768131.251053:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768131.251054:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207889f80 x1862022294278272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768131.251058:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768131.251059:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768131.251061:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768131.251062:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768131.251063:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768131.251065:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207889f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294278272:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768131.251067:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768131.251068:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768131.251069:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768131.251070:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768131.251072:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1e400. 02000000:00000001:2.0:1775768131.251073:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768131.251075:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768131.251076:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768131.251078:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768131.251080:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294278272, portal 4 00000100:00000001:2.0:1775768131.251081:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768131.251082:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606126464 : -112966103425152 : ffff994207889f80) 00000100:00000040:2.0:1775768131.251085:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207889f80 x1862022294278272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768156 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768131.251090:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768131.251091:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768131.251092:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768131.251093:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294278272, offset 0 00000400:00000200:2.0:1775768131.251095:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768131.251098:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768131.251099:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768131.251101:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768131.251103:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768131.251104:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768131.251105:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768131.251107:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225347:225347:256:4294967295] 192.168.202.156@tcp LPNI seq info [225343:225343:8:4294967295] 00000400:00000200:2.0:1775768131.251110:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768131.251113:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768131.251116:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768131.251118:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768131.251120:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768131.251121:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1775768131.251124:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768131.251125:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000100:2.0:1775768131.251127:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768131.251128:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768131.251130:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768131.251132:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768131.251134:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768131.251135:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768131.251136:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768131.251138:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207889f80 x1862022294278272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768156 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768131.251142:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768131.251144:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207889f80 x1862022294278272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768156 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768131.251151:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768131.251152:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768131.251153:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768131.251170:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:2.0:1775768131.251172:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768131.251173:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300056102 00000100:00000001:2.0:1775768131.251175:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300057405 00000100:00000001:2.0:1775768131.251177:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300056102 00000100:00000001:2.0:1775768131.251178:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300057405 00000100:00000001:2.0:1775768131.251179:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768131.251180:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768131.251181:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768131.251183:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768131/real 1775768131] req@ffff994207889f80 x1862022294278272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768131.251187:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768131.251188:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768131.251189:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768131.251190:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768131.251191:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768131.251192:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768131.251194:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207889f80 x1862022294278272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768131.251198:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207889f80 x1862022294278272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768131.251202:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768131.251203:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768131.251204:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768131.251205:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768131.251206:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207889f80 x1862022294278272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768131.251210:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768131.251212:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207889f80 x1862022294278272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768156 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768131.251216:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768131.251217:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768131.251218:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768131.251219:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768131.251220:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768131.251222:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768131.251223:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768131.251225:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768131.251226:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768131.251227:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207889f80 x1862022294278272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768156 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768131.251232:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207889f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294278272:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768131.251235:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768131.251237:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768131.251239:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207889f80 x1862022294278272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768156 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768131.251245:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768131.251247:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768131.251248:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1e400. 02000000:00000001:2.0:1775768131.251250:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768131.251251:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768131.251252:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:2.0:1775768131.251254:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768131.251256:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543c400. 02000000:00000001:2.0:1775768131.251258:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768131.251259:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768131.251261:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207889f80. 00000100:00000001:2.0:1775768131.251263:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768131.251264:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768131.251266:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768131.251269:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768131.251270:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768131.251271:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768131.251273:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768131.313232:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768131.313236:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768131.313240:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768131.313241:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768131.313244:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768131.313245:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768131.313246:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768131.313247:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768131.313249:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768131.313251:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768131.313292:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768131.313294:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768131.313297:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768131.377214:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768131.377219:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768131.377221:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768131.377225:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768131.377227:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768131.377230:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768131.377231:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768131.377233:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768131.377234:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768132.273233:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768132.273236:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768132.273242:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768132.273242:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768132.273245:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768132.273246:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768132.273248:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:0.0:1775768132.273252:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:2.0:1775768132.273252:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775768132.273252:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768132.273255:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768132.273258:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1775768132.273258:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768132.273259:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768132.273261:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768132.273262:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768132.273263:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768132.273265:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768132.273265:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768132.273268:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768132.273271:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768132.273274:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768132.273277:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768132.273281:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768132.273281:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768132.273283:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768132.273287:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768132.337262:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768132.337269:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768132.337271:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768132.337273:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768132.337277:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768132.337279:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768132.337280:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768132.337282:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768132.337285:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768132.337286:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768132.337288:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768132.337289:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768132.337291:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768132.337291:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768132.337293:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768132.337294:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768132.337347:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768132.337351:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768132.337355:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768132.401250:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768132.401255:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768132.401257:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768132.401261:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768132.401263:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768132.401266:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768132.401268:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768132.401269:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768132.401271:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768132.593256:0:14953:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4300057405, at 4300057408 00000100:00000010:0.0:1775768132.593268:0:14953:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d6a20. 00000100:00000040:1.0:1775768132.593273:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4300057405, at 4300057408 00000100:00000010:1.0:1775768132.593285:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d6ea0. 00000100:00000001:3.0:1775768133.297236:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768133.297245:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768133.297246:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768133.297250:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768133.297250:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768133.297253:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768133.297259:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775768133.297260:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768133.297264:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768133.297265:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768133.297270:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768133.297270:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768133.297276:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768133.297279:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768133.297280:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768133.297287:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768133.297287:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768133.297288:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768133.297290:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768133.297295:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768133.297296:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768133.297306:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:0.0:1775768133.297310:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768133.297315:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768133.297319:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768133.297321:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768133.297324:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768133.361287:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768133.361293:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768133.361298:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768133.361300:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768133.361303:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768133.361305:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768133.361309:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768133.361312:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768133.361317:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768133.361319:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768133.361376:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768133.361379:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768133.361382:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768133.425267:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768133.425272:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768133.425274:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768133.425279:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768133.425282:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768133.425294:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768133.425296:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768133.425301:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768133.425305:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768134.321238:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768134.321243:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768134.321244:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768134.321249:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768134.321249:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768134.321252:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768134.321258:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775768134.321258:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768134.321263:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768134.321264:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768134.321271:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768134.321271:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768134.321278:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768134.321278:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768134.321282:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768134.321287:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768134.321288:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768134.321288:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768134.321291:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768134.321292:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768134.321298:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768134.321301:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768134.321306:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768134.321311:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768134.321314:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768134.321317:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768134.321321:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768134.385262:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768134.385268:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768134.385274:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768134.385284:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768134.385288:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768134.385289:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768134.385293:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768134.385294:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768134.385298:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768134.385299:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768134.385380:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768134.385383:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768134.385385:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768134.449269:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768134.449283:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768134.449285:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768134.449291:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768134.449294:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768134.449298:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768134.449300:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768134.449303:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768134.449306:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768135.345226:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768135.345230:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768135.345233:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768135.345234:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768135.345236:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768135.345237:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768135.345243:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775768135.345244:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768135.345249:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768135.345249:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768135.345253:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768135.345253:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768135.345259:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768135.345259:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768135.345260:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768135.345264:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768135.345265:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768135.345265:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768135.345268:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768135.345269:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768135.345271:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768135.345273:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:0.0:1775768135.345277:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768135.345280:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768135.345282:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768135.345284:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768135.345286:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768135.409248:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768135.409252:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768135.409255:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768135.409257:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768135.409261:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768135.409262:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768135.409266:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768135.409267:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768135.409271:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768135.409272:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768135.409327:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768135.409331:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768135.409334:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768135.473244:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768135.473250:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768135.473251:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768135.473255:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768135.473257:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768135.473260:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768135.473262:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768135.473264:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768135.473266:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768136.369220:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768136.369228:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768136.369230:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768136.369235:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768136.369239:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768136.369243:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768136.369246:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768136.369248:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768136.369251:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768136.369255:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768136.369257:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768136.369259:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775768136.369259:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768136.369262:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768136.369265:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768136.369265:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768136.369268:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768136.369269:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768136.369271:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768136.369271:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768136.369272:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775768136.369276:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768136.369276:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768136.369282:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1775768136.369282:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775768136.369285:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768136.369287:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768136.369287:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775768136.369290:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768136.369292:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768136.369292:0:36:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775768136.369298:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775768136.369305:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5388 00000400:00000200:3.0:1775768136.369315:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775768136.369322:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775768136.369326:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775768136.369329:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768136.369332:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775768136.369335:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768136.369337:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775768136.369341:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775768136.369343:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:3.0:1775768136.369345:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775768136.369347:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:3.0:1775768136.369349:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:3.0:1775768136.369351:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:3.0:1775768136.369364:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768136.369366:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775768136.369373:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768136.369376:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775768136.369382:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:3.0:1775768136.369385:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775768136.369388:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768136.369390:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775768136.369393:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775768136.369396:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768136.369399:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775768136.369401:0:36:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775768136.369409:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8a80. 00000020:00000040:3.0:1775768136.369412:0:36:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:3.0:1775768136.369416:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775768136.369418:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768136.369420:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775768136.369424:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775768136.369428:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775768136.369437:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2800. 00000100:00000001:3.0:1775768136.369444:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768136.369453:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8a80 x1862022294278400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775768136.369462:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775768136.369467:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768136.369471:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006c3265af] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8a80 x1862022294278400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768136.369493:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768136.369496:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768136.369499:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775768136.369500:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775768136.369502:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775768136.369505:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775768136.369507:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5388 00000400:00000200:3.0:1775768136.369511:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775768136.369515:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775768136.369518:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775768136.369521:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768136.369524:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775768136.369526:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768136.369528:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775768136.369530:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775768136.369532:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000100:00000001:2.0:1775768136.369534:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1775768136.369534:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775768136.369538:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000100:00000001:2.0:1775768136.369539:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1775768136.369541:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000001:2.0:1775768136.369542:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1775768136.369546:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:2.0:1775768136.369552:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a8a80 x1862022294278400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775768136.369552:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768136.369563:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768136.369565:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768136.369565:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775768136.369569:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000001:2.0:1775768136.369571:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1775768136.369572:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 02000000:00000001:2.0:1775768136.369593:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768136.369597:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768136.369601:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294278400:192.168.202.156@tcp:250:kworker.0 00000100:00000001:2.0:1775768136.369605:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768136.369609:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:3.0:1775768136.369609:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 02000000:00000001:2.0:1775768136.369612:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768136.369615:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 02000000:00000001:2.0:1775768136.369616:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1775768136.369620:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775768136.369623:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1e400. 02000000:00000001:3.0:1775768136.369623:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775768136.369626:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768136.369628:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775768136.369632:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768136.369634:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775768136.369635:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000010:2.0:1775768136.369639:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000008:00100000:3.0:1775768136.369641:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775768136.369645:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775768136.369647:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000001:3.0:1775768136.369647:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775768136.369654:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aa300. 00000020:00000040:3.0:1775768136.369657:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000100:00000200:2.0:1775768136.369658:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294278400, portal 25 02000000:00000001:3.0:1775768136.369661:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1775768136.369662:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1775768136.369664:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768136.369665:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640855168 : -112966068696448 : ffff9942099a8a80) 00000100:00000001:3.0:1775768136.369667:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775768136.369669:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775768136.369671:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775768136.369675:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2c00. 00000100:00000001:3.0:1775768136.369680:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768136.369686:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a8a80 x1862022294278400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768161 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768136.369689:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aa300 x1862022294278528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775768136.369698:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768136.369703:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768136.369706:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768136.369710:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294278400, offset 0 00000400:00000200:2.0:1775768136.369716:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768136.369723:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768136.369729:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768136.369735:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1775768136.369745:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775768136.369747:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768136.369751:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000100:00000040:3.0:1775768136.369752:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f33bdf15] to pc [ptlrpcd_rcv+-1] req@ffff9942099aa300 x1862022294278528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1775768136.369757:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768136.369762:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768136.369766:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225348:225348:256:4294967295] 192.168.202.156@tcp LPNI seq info [225344:225344:8:4294967295] 00000400:00000200:2.0:1775768136.369771:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768136.369784:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000100:00000001:3.0:1775768136.369788:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768136.369792:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:2.0:1775768136.369793:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00080000:3.0:1775768136.369798:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775768136.369800:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000800:00000010:2.0:1775768136.369801:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000100:00080000:3.0:1775768136.369801:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775768136.369806:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775768136.369809:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5388 00000800:00000200:2.0:1775768136.369813:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000400:00000200:3.0:1775768136.369816:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000800:00000200:2.0:1775768136.369819:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000400:00000200:3.0:1775768136.369822:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775768136.369825:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000800:00000200:2.1:1775768136.369829:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000040:3.0:1775768136.369830:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768136.369835:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:2.1:1775768136.369835:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000100:00000040:3.0:1775768136.369838:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768136.369841:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775768136.369844:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000800:00000100:2.0:1775768136.369845:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000020:00000040:3.0:1775768136.369846:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:3.0:1775768136.369848:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000800:00000010:2.0:1775768136.369850:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000020:00000040:3.0:1775768136.369851:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:3.0:1775768136.369855:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:3.0:1775768136.369857:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000400:00000200:2.0:1775768136.369858:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:3.0:1775768136.369862:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768136.369865:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1775768136.369867:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000100:00000040:3.0:1775768136.369869:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768136.369872:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000200:2.0:1775768136.369874:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000020:00000040:3.0:1775768136.369874:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:3.0:1775768136.369877:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000400:00000010:2.0:1775768136.369879:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:3.0:1775768136.369880:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768136.369883:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1775768136.369884:0:2399:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:3.0:1775768136.369885:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775768136.369889:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768136.369891:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000200:2.0:1775768136.369892:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a8a80 x1862022294278400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768161 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1775768136.369893:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775768136.369899:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768136.369903:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775768136.369909:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ab800. 00000100:00000001:2.0:1775768136.369910:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000020:00000040:3.0:1775768136.369913:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000100:00000040:2.0:1775768136.369915:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a8a80 x1862022294278400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768161 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1775768136.369917:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775768136.369920:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768136.369924:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775768136.369926:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1775768136.369929:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768136.369929:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768136.369934:0:2399:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000010:3.0:1775768136.369936:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0000. 00000100:00000001:2.0:1775768136.369940:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768136.369941:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768136.369946:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768136.369949:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4300062525 00000100:00000010:2.0:1775768136.369955:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d6ea0. 00000100:00100000:3.0:1775768136.369962:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ab800 x1862022294278656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775768136.369963:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1775768136.369970:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4300062525 00000100:00000001:3.0:1775768136.369971:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000010:2.0:1775768136.369974:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d6a20. 00000100:00000001:3.0:1775768136.369975:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768136.369980:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000040:3.0:1775768136.369982:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000089df80c0] to pc [ptlrpcd_rcv+-1] req@ffff9942099ab800 x1862022294278656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768136.369984:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768136.369989:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768136.369994:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1775768136.369999:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768136.370002:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768136/real 1775768136] req@ffff9942099a8a80 x1862022294278400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768136.370003:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768136.370010:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775768136.370014:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775768136.370016:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775768136.370020:0:36:0:(import.c:484:import_select_connection()) Process entered 00000400:00000001:2.0:1775768136.370021:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768136.370024:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:3.0:1775768136.370024:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5388 00000400:00000010:2.0:1775768136.370028:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:3.0:1775768136.370031:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768136.370033:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768136.370037:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768136.370040:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000200:3.0:1775768136.370041:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000200:2.0:1775768136.370045:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a8a80 x1862022294278400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768136.370045:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775768136.370051:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768136.370055:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775768136.370060:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768136.370063:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000200:2.0:1775768136.370066:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a8a80 x1862022294278400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775768136.370067:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775768136.370070:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:3.0:1775768136.370074:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775768136.370077:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000100:00000001:2.0:1775768136.370078:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:3.0:1775768136.370080:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:2.0:1775768136.370082:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1775768136.370083:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:2.0:1775768136.370086:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:3.0:1775768136.370089:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768136.370090:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768136.370092:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:2.0:1775768136.370097:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a8a80 x1862022294278400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775768136.370097:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768136.370100:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775768136.370105:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:3.0:1775768136.370110:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775768136.370114:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768136.370115:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768136.370119:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775768136.370122:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1775768136.370123:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a8a80 x1862022294278400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768161 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1775768136.370126:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768136.370130:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775768136.370133:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:3.0:1775768136.370138:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768136.370140:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775768136.370141:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768136.370145:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:3.0:1775768136.370146:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aa680. 00000100:00000001:2.0:1775768136.370149:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000040:3.0:1775768136.370150:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00000100:00000001:2.0:1775768136.370153:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768136.370178:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768136.370180:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768136.370183:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:3.0:1775768136.370186:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00080000:2.0:1775768136.370189:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 02000000:00000001:3.0:1775768136.370189:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768136.370194:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1775768136.370195:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:3.0:1775768136.370196:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775768136.370199:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768136.370203:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a8a80 x1862022294278400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768161 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1775768136.370205:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0400. 00000100:00000001:3.0:1775768136.370211:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768136.370219:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294278400:192.168.202.156@tcp:250:kworker.0 00000100:00100000:3.0:1775768136.370220:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aa680 x1862022294278784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775768136.370228:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768136.370232:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1775768136.370234:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775768136.370237:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768136.370239:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a8a80 x1862022294278400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768161 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775768136.370245:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006cb1597d] to pc [ptlrpcd_rcv+-1] req@ffff9942099aa680 x1862022294278784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768136.370251:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768136.370255:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768136.370259:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1e400. 00000100:00000001:3.0:1775768136.370259:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768136.370264:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:2.0:1775768136.370266:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768136.370270:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000100:00080000:3.0:1775768136.370270:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775768136.370272:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000020:00000040:2.0:1775768136.370274:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:2.0:1775768136.370276:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768136.370278:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b2800. 02000000:00000001:2.0:1775768136.370281:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768136.370282:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768136.370283:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a8a80. 00000100:00000001:2.0:1775768136.370285:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768136.370286:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768136.370288:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768136.370302:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768136.370307:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768136.370309:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768136.370311:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768136.370312:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768136.370316:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099aa300 x1862022294278528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768136.370323:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768136.370324:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768136.370326:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768136.370328:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768136.370330:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768136.370332:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099aa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294278528:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768136.370336:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768136.370338:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768136.370339:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768136.370341:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768136.370344:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1e400. 02000000:00000001:2.0:1775768136.370346:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768136.370350:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768136.370358:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768136.370362:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768136.370365:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294278528, portal 10 00000100:00000001:2.0:1775768136.370367:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768136.370369:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640861440 : -112966068690176 : ffff9942099aa300) 00000100:00000040:2.0:1775768136.370374:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099aa300 x1862022294278528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768161 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768136.370380:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768136.370382:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768136.370384:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768136.370386:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294278528, offset 0 00000400:00000200:2.0:1775768136.370390:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768136.370394:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768136.370398:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768136.370400:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768136.370402:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768136.370404:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768136.370406:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768136.370409:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225349:225349:256:4294967295] 192.168.202.156@tcp LPNI seq info [225345:225345:8:4294967295] 00000400:00000200:2.0:1775768136.370413:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768136.370419:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768136.370423:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768136.370426:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768136.370430:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768136.370432:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1775768136.370437:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768136.370439:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000100:2.0:1775768136.370443:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768136.370444:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768136.370447:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768136.370449:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768136.370451:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768136.370452:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768136.370454:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768136.370456:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099aa300 x1862022294278528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768161 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768136.370461:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768136.370463:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099aa300 x1862022294278528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768161 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768136.370471:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768136.370472:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768136.370473:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768136.370476:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:2.0:1775768136.370478:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768136.370480:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300061203 00000100:00000001:2.0:1775768136.370481:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300062525 00000100:00000001:2.0:1775768136.370483:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300061203 00000100:00000001:2.0:1775768136.370484:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300062525 00000100:00000001:2.0:1775768136.370485:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768136.370486:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768136.370488:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768136.370491:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768136/real 1775768136] req@ffff9942099aa300 x1862022294278528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768136.370498:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768136.370499:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768136.370500:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768136.370502:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768136.370504:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768136.370506:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768136.370508:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099aa300 x1862022294278528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768136.370515:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099aa300 x1862022294278528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768136.370521:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768136.370523:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768136.370524:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768136.370526:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768136.370529:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099aa300 x1862022294278528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768136.370535:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768136.370539:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099aa300 x1862022294278528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768161 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768136.370545:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768136.370548:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768136.370549:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768136.370550:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768136.370552:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768136.370554:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768136.370557:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768136.370560:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768136.370562:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768136.370566:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099aa300 x1862022294278528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768161 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768136.370573:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099aa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294278528:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768136.370589:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768136.370592:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ab800 x1862022294278656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768136.370598:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768136.370600:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768136.370601:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768136.370603:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768136.370605:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768136.370608:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294278656:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768136.370611:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768136.370613:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768136.370614:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768136.370615:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768136.370617:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f000. 02000000:00000001:2.0:1775768136.370619:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768136.370621:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768136.370623:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768136.370626:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768136.370629:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294278656, portal 10 00000100:00000001:2.0:1775768136.370631:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768136.370632:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640866816 : -112966068684800 : ffff9942099ab800) 00000100:00000040:2.0:1775768136.370637:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ab800 x1862022294278656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768161 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768136.370643:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768136.370645:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768136.370647:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768136.370649:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294278656, offset 0 00000400:00000200:2.0:1775768136.370652:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768136.370656:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768136.370660:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768136.370662:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768136.370665:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768136.370667:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768136.370669:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768136.370672:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225350:225350:256:4294967295] 192.168.202.156@tcp LPNI seq info [225346:225346:8:4294967295] 00000400:00000200:2.0:1775768136.370676:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768136.370683:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768136.370687:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768136.370690:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768136.370694:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768136.370697:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1775768136.370702:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768136.370705:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000100:2.0:1775768136.370709:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768136.370711:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768136.370713:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768136.370717:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768136.370720:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768136.370722:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768136.370724:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768136.370727:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ab800 x1862022294278656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768161 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768136.370735:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768136.370738:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ab800 x1862022294278656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768161 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768136.370744:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768136.370746:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768136.370748:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768136.370751:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:2.0:1775768136.370754:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768136.370756:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300061195 00000100:00000001:2.0:1775768136.370758:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300062525 00000100:00000001:2.0:1775768136.370760:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300061195 00000100:00000001:2.0:1775768136.370763:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300062525 00000100:00000001:2.0:1775768136.370765:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768136.370767:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768136.370768:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768136.370772:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768136/real 1775768136] req@ffff9942099ab800 x1862022294278656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768136.370780:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768136.370781:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768136.370783:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768136.370785:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768136.370786:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768136.370788:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768136.370792:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ab800 x1862022294278656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768136.370800:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ab800 x1862022294278656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768136.370806:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768136.370807:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768136.370809:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768136.370810:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768136.370814:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ab800 x1862022294278656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768136.370820:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768136.370823:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ab800 x1862022294278656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768161 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768136.370830:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768136.370832:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768136.370834:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768136.370835:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768136.370837:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768136.370840:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768136.370843:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768136.370846:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768136.370848:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768136.370851:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ab800 x1862022294278656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768161 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768136.370859:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294278656:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768136.370863:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768136.370866:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099aa680 x1862022294278784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768136.370872:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768136.370874:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768136.370876:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768136.370878:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768136.370880:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768136.370882:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099aa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294278784:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768136.370885:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768136.370887:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768136.370889:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768136.370890:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768136.370894:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f800. 02000000:00000001:2.0:1775768136.370896:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768136.370898:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768136.370901:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768136.370904:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768136.370907:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294278784, portal 4 00000100:00000001:2.0:1775768136.370909:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768136.370911:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640862336 : -112966068689280 : ffff9942099aa680) 00000100:00000040:2.0:1775768136.370915:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099aa680 x1862022294278784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768161 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768136.370921:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768136.370923:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768136.370925:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768136.370928:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294278784, offset 0 00000400:00000200:2.0:1775768136.370932:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768136.370935:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768136.370939:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768136.370941:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768136.370943:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768136.370946:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768136.370947:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768136.370951:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225351:225351:256:4294967295] 192.168.202.156@tcp LPNI seq info [225347:225347:8:4294967295] 00000400:00000200:2.0:1775768136.370955:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768136.370960:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768136.370964:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768136.370967:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768136.370970:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768136.370973:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1775768136.370976:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768136.370979:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000100:2.0:1775768136.370982:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768136.370984:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768136.370987:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768136.370990:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768136.370993:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768136.370994:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768136.370996:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:3.0:1775768136.370996:0:36:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:2.0:1775768136.371001:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099aa680 x1862022294278784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768161 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768136.371002:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775768136.371008:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5388 00000100:00000001:2.0:1775768136.371014:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000400:00000200:3.0:1775768136.371016:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000040:2.0:1775768136.371020:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099aa680 x1862022294278784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768161 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1775768136.371022:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775768136.371027:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775768136.371034:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768136.371035:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768136.371038:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775768136.371038:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775768136.371043:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768136.371044:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768136.371046:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000040:2.0:1775768136.371048:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000020:00000001:3.0:1775768136.371050:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775768136.371052:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000400:00000200:2.0:1775768136.371055:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:3.0:1775768136.371055:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:2.0:1775768136.371059:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300061218 00000020:00000040:3.0:1775768136.371059:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000100:00000001:2.0:1775768136.371063:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300062525 00000020:00000001:3.0:1775768136.371063:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:3.0:1775768136.371066:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000001:2.0:1775768136.371067:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300061218 00000100:00000001:2.0:1775768136.371073:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300062525 00000100:00000040:3.0:1775768136.371073:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768136.371077:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775768136.371078:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1775768136.371081:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768136.371083:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768136.371084:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:2.0:1775768136.371086:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000020:00000040:3.0:1775768136.371088:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00100000:2.0:1775768136.371093:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768136/real 1775768136] req@ffff9942099aa680 x1862022294278784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768136.371093:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775768136.371098:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768136.371104:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000001:2.0:1775768136.371108:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:3.0:1775768136.371109:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000001:2.0:1775768136.371110:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768136.371113:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 02000000:00000001:3.0:1775768136.371114:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768136.371117:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000100:00000001:3.0:1775768136.371117:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000010:2.0:1775768136.371120:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000008:00100000:3.0:1775768136.371120:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000100:00000001:2.0:1775768136.371124:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000008:00000001:3.0:1775768136.371125:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768136.371128:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1775768136.371130:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099aa680 x1862022294278784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1775768136.371133:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aad80. 00000020:00000040:3.0:1775768136.371137:0:36:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:3.0:1775768136.371142:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775768136.371144:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1775768136.371145:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099aa680 x1862022294278784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768136.371147:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775768136.371150:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775768136.371153:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768136.371179:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768136.371181:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768136.371185:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768136.371187:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:3.0:1775768136.371191:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b3400. 00000100:00100000:2.0:1775768136.371192:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099aa680 x1862022294278784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768136.371197:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768136.371199:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768136.371205:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099aa680 x1862022294278784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768161 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768136.371206:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aad80 x1862022294278912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775768136.371222:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775768136.371226:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775768136.371227:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768136.371230:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775768136.371231:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768136.371233:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768136.371236:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000040:3.0:1775768136.371238:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000045ccf08] to pc [ptlrpcd_rcv+-1] req@ffff9942099aad80 x1862022294278912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768136.371241:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768136.371248:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1775768136.371253:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768136.371254:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768136.371259:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775768136.371261:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5398) 00000100:00000040:2.0:1775768136.371265:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099aa680 x1862022294278784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768161 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768136.371274:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099aa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294278784:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768136.371278:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768136.371279:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768136.371282:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099aa300 x1862022294278528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768161 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768136.371286:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768136.371288:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768136.371289:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1e400. 02000000:00000001:2.0:1775768136.371292:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768136.371294:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768136.371295:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:2.0:1775768136.371297:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768136.371299:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b2c00. 02000000:00000001:2.0:1775768136.371301:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768136.371303:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768136.371304:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099aa300. 00000100:00000001:2.0:1775768136.371306:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768136.371307:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768136.371308:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768136.371311:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ab800 x1862022294278656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768161 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768136.371317:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768136.371318:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768136.371319:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f000. 02000000:00000001:2.0:1775768136.371321:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768136.371323:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768136.371324:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:2.0:1775768136.371326:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768136.371327:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b0000. 02000000:00000001:2.0:1775768136.371330:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768136.371331:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768136.371332:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ab800. 00000100:00000001:2.0:1775768136.371335:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768136.371336:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768136.371337:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768136.371340:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099aa680 x1862022294278784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768161 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768136.371345:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768136.371346:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768136.371347:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f800. 02000000:00000001:2.0:1775768136.371348:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768136.371349:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768136.371349:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:2.0:1775768136.371351:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768136.371351:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b0400. 02000000:00000001:2.0:1775768136.371353:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768136.371353:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768136.371354:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099aa680. 00000100:00000001:2.0:1775768136.371355:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768136.371356:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768136.371357:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768136.371361:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768136.371361:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768136.371363:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768136.371363:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768136.371364:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768136.371366:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099aad80 x1862022294278912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768136.371370:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768136.371371:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768136.371372:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768136.371373:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768136.371374:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768136.371376:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099aad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294278912:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768136.371378:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768136.371380:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768136.371380:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768136.371381:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768136.371383:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f800. 02000000:00000001:2.0:1775768136.371384:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768136.371387:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768136.371388:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768136.371390:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768136.371392:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294278912, portal 4 00000100:00000001:2.0:1775768136.371393:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768136.371394:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640864128 : -112966068687488 : ffff9942099aad80) 00000100:00000040:2.0:1775768136.371397:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099aad80 x1862022294278912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768161 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768136.371401:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768136.371402:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768136.371404:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768136.371405:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294278912, offset 0 00000400:00000200:2.0:1775768136.371407:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768136.371410:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768136.371412:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768136.371414:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768136.371415:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768136.371417:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768136.371419:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768136.371420:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225352:225352:256:4294967295] 192.168.202.156@tcp LPNI seq info [225348:225348:8:4294967295] 00000400:00000200:2.0:1775768136.371423:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768136.371427:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768136.371429:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768136.371431:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768136.371434:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768136.371435:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1775768136.371438:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768136.371439:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 28 secs later) 00000800:00000100:2.0:1775768136.371442:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768136.371443:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768136.371444:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768136.371446:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768136.371448:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768136.371449:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768136.371450:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768136.371452:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099aad80 x1862022294278912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768161 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768136.371456:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768136.371458:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099aad80 x1862022294278912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768161 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768136.371462:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768136.371463:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768136.371464:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768136.371466:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:2.0:1775768136.371467:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768136.371469:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300061222 00000100:00000001:2.0:1775768136.371470:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300062525 00000100:00000001:2.0:1775768136.371471:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300061222 00000100:00000001:2.0:1775768136.371473:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300062525 00000100:00000001:2.0:1775768136.371474:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768136.371475:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768136.371476:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768136.371478:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768136/real 1775768136] req@ffff9942099aad80 x1862022294278912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768136.371482:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768136.371483:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768136.371484:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768136.371485:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768136.371486:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768136.371487:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768136.371489:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099aad80 x1862022294278912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768136.371493:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099aad80 x1862022294278912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768136.371496:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768136.371497:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768136.371498:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768136.371498:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768136.371500:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099aad80 x1862022294278912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768136.371503:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768136.371505:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099aad80 x1862022294278912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768161 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768136.371509:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768136.371510:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768136.371511:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768136.371512:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768136.371513:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768136.371515:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768136.371517:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768136.371518:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768136.371519:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768136.371521:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099aad80 x1862022294278912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768161 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768136.371525:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099aad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294278912:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768136.371528:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768136.371529:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768136.371530:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099aad80 x1862022294278912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768161 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768136.371534:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768136.371534:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768136.371536:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f800. 02000000:00000001:2.0:1775768136.371537:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768136.371538:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768136.371539:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:2.0:1775768136.371540:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768136.371542:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b3400. 02000000:00000001:2.0:1775768136.371544:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768136.371545:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768136.371546:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099aad80. 00000100:00000001:2.0:1775768136.371548:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768136.371549:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768136.371551:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768136.371553:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768136.371555:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768136.371556:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768136.371557:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768136.433245:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768136.433251:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768136.433257:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768136.433258:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768136.433262:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768136.433263:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768136.433266:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768136.433268:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768136.433271:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768136.433272:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768136.433314:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768136.433317:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768136.433319:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768136.497251:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768136.497256:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768136.497258:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768136.497264:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768136.497267:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768136.497271:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768136.497274:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768136.497276:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768136.497279:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768137.393216:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768137.393217:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768137.393222:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768137.393222:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768137.393224:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768137.393224:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768137.393229:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:1.0:1775768137.393229:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768137.393233:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768137.393233:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768137.393234:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768137.393238:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768137.393238:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768137.393240:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768137.393243:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768137.393244:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768137.393245:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768137.393247:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768137.393248:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768137.393251:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768137.393252:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768137.393255:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768137.393257:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768137.393260:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768137.393268:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768137.393270:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768137.393273:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768137.457268:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768137.457272:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768137.457277:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768137.457279:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768137.457281:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768137.457283:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768137.457286:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768137.457288:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768137.457290:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768137.457292:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768137.457296:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768137.457297:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768137.457298:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768137.457300:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768137.457301:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768137.457302:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768137.457304:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768137.457305:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768137.457309:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768137.457310:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768137.457311:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768137.457312:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768137.457314:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768137.457315:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768137.457316:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768137.457318:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768137.457321:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768137.457323:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768137.457324:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768137.457325:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768137.457327:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768137.457341:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768137.457343:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768137.457345:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768137.457397:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768137.457401:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768137.457404:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768137.521240:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768137.521245:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768137.521246:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768137.521250:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768137.521253:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768137.521256:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768137.521257:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768137.521259:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768137.521261:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768137.713229:0:14953:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4300062525, at 4300062528 00000100:00000040:1.0:1775768137.713241:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4300062525, at 4300062528 00000100:00000010:0.0:1775768137.713251:0:14953:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d6ea0. 00000100:00000010:1.0:1775768137.713253:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d6a20. 00000100:00000001:1.0:1775768138.417225:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768138.417225:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768138.417231:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768138.417231:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768138.417232:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768138.417234:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768138.417236:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768138.417239:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768138.417239:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768138.417244:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768138.417245:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775768138.417245:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768138.417249:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768138.417250:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1775768138.417250:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768138.417252:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768138.417255:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768138.417255:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768138.417255:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768138.417260:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768138.417260:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768138.417260:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768138.417263:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768138.417266:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768138.417267:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768138.417269:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768138.417271:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768138.481237:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768138.481244:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768138.481250:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768138.481251:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768138.481255:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768138.481256:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768138.481259:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768138.481260:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768138.481263:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768138.481264:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768138.481321:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768138.481324:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768138.481326:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768138.545214:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768138.545219:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768138.545220:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768138.545224:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768138.545227:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768138.545231:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768138.545232:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768138.545234:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768138.545236:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768139.441238:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768139.441238:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768139.441240:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768139.441244:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768139.441244:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768139.441244:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768139.441247:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768139.441247:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768139.441248:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768139.441253:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775768139.441253:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:0.0:1775768139.441254:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768139.441259:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768139.441260:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768139.441260:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768139.441262:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768139.441264:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768139.441264:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768139.441265:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768139.441267:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768139.441270:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768139.441271:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768139.441273:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768139.441275:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768139.441278:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768139.441280:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768139.441283:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768139.505245:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768139.505250:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768139.505256:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768139.505257:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768139.505261:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768139.505262:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768139.505265:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768139.505266:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768139.505269:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768139.505270:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768139.505320:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768139.505322:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768139.505325:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768139.569226:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768139.569233:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768139.569235:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768139.569240:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768139.569244:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768139.569247:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768139.569249:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768139.569251:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768139.569253:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768140.465225:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768140.465231:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768140.465232:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768140.465236:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768140.465236:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:2.0:1775768140.465237:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768140.465241:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768140.465242:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768140.465242:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768140.465244:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768140.465246:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768140.465246:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768140.465248:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00100000:1.0:1775768140.465251:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768140.465251:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768140.465252:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775768140.465255:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768140.465257:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768140.465257:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768140.465260:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768140.465261:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768140.465265:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768140.465268:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768140.465270:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768140.465271:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768140.465274:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768140.465276:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768140.529237:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768140.529241:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768140.529244:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768140.529245:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768140.529247:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768140.529248:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768140.529250:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768140.529251:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768140.529253:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768140.529253:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768140.529297:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768140.529299:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768140.529302:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768140.593250:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768140.593257:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768140.593258:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768140.593264:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768140.593268:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768140.593272:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768140.593274:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768140.593276:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768140.593278:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768141.489260:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768141.489260:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768141.489261:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768141.489267:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768141.489267:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768141.489267:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768141.489269:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768141.489270:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768141.489271:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768141.489274:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775768141.489274:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:1.0:1775768141.489276:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768141.489279:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1775768141.489280:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1775768141.489280:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768141.489281:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768141.489283:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768141.489284:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768141.489286:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768141.489289:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768141.489291:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1775768141.489292:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775768141.489294:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768141.489294:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768141.489296:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1775768141.489297:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768141.489300:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768141.489301:0:36:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768141.489301:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768141.489304:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768141.489306:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775768141.489308:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768141.489313:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5393 00000400:00000200:0.0:1775768141.489322:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768141.489328:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768141.489333:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768141.489341:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768141.489345:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768141.489348:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768141.489351:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768141.489355:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768141.489357:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:0.0:1775768141.489360:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768141.489363:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:0.0:1775768141.489366:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:0.0:1775768141.489369:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:0.0:1775768141.489372:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768141.489375:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768141.489377:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768141.489380:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768141.489382:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:0.0:1775768141.489385:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768141.489387:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768141.489389:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768141.489391:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768141.489393:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768141.489395:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775768141.489397:0:36:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768141.489406:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fc000. 00000020:00000040:0.0:1775768141.489409:0:36:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:0.0:1775768141.489412:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768141.489413:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768141.489416:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768141.489419:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768141.489424:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768141.489431:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220029400. 00000100:00000001:0.0:1775768141.489436:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768141.489444:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fc000 x1862022294279040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768141.489451:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768141.489454:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768141.489457:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008bcb9792] to pc [ptlrpcd_rcv+-1] req@ffff9942099fc000 x1862022294279040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768141.489473:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768141.489477:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768141.489479:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775768141.489481:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768141.489483:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768141.489486:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768141.489489:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5393 00000100:00000001:2.0:1775768141.489494:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768141.489498:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768141.489500:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000400:00000200:0.0:1775768141.489502:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768141.489508:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000040:2.0:1775768141.489509:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fc000 x1862022294279040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768141.489510:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768141.489516:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768141.489518:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768141.489520:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768141.489522:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768141.489535:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768141.489536:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:0.0:1775768141.489538:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768141.489541:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:0.0:1775768141.489545:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:0.0:1775768141.489547:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000001:2.0:1775768141.489547:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768141.489551:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768141.489552:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768141.489554:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768141.489556:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000001:2.0:1775768141.489556:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1775768141.489557:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768141.489560:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 02000000:00000001:2.0:1775768141.489560:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:0.0:1775768141.489564:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768141.489566:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768141.489566:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768141.489570:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:2.0:1775768141.489572:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294279040:192.168.202.156@tcp:250:kworker.0 02000000:00000001:0.0:1775768141.489574:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768141.489578:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768141.489579:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1775768141.489580:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775768141.489584:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1775768141.489584:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000008:00000001:0.0:1775768141.489587:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768141.489587:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768141.489591:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768141.489591:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000010:0.0:1775768141.489595:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ff800. 02000000:00000010:2.0:1775768141.489596:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f800. 00000020:00000040:0.0:1775768141.489598:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:2.0:1775768141.489599:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768141.489601:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768141.489603:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768141.489604:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:2.0:1775768141.489606:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:0.0:1775768141.489607:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768141.489610:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775768141.489611:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 02000000:00000010:0.0:1775768141.489614:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002bc00. 00000400:00000010:2.0:1775768141.489617:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000001:0.0:1775768141.489618:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768141.489623:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ff800 x1862022294279168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:2.0:1775768141.489623:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294279040, portal 25 00000100:00000001:2.0:1775768141.489627:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768141.489630:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641196544 : -112966068355072 : ffff9942099fc000) 00000100:00000001:0.0:1775768141.489635:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768141.489638:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768141.489639:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fc000 x1862022294279040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768166 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775768141.489643:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000aa4e8b71] to pc [ptlrpcd_rcv+-1] req@ffff9942099ff800 x1862022294279168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768141.489649:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768141.489655:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768141.489658:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768141.489662:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294279040, offset 0 00000100:00000001:0.0:1775768141.489664:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768141.489668:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00080000:0.0:1775768141.489669:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768141.489673:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775768141.489675:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768141.489676:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768141.489679:0:36:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:2.0:1775768141.489682:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00080000:0.0:1775768141.489684:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5393 00000400:00000200:2.0:1775768141.489689:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768141.489690:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768141.489694:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768141.489697:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000400:00000200:2.0:1775768141.489699:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000100:00080000:0.0:1775768141.489702:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1775768141.489704:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000100:00000040:0.0:1775768141.489710:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000200:2.0:1775768141.489710:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00000001:0.0:1775768141.489714:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1775768141.489716:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225353:225353:256:4294967295] 192.168.202.156@tcp LPNI seq info [225349:225349:8:4294967295] 00000100:00000040:0.0:1775768141.489719:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768141.489724:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768141.489728:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000400:00000200:2.0:1775768141.489728:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000040:0.0:1775768141.489730:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:0.0:1775768141.489733:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768141.489735:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000400:00000200:2.0:1775768141.489742:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000020:00000001:0.0:1775768141.489744:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:0.0:1775768141.489748:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000800:00000200:2.0:1775768141.489753:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00000040:0.0:1775768141.489757:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000800:00000010:2.0:1775768141.489758:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000100:00000001:0.0:1775768141.489760:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768141.489764:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768141.489766:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768141.489770:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000800:00000200:2.0:1775768141.489771:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00080000:0.0:1775768141.489776:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000800:00000200:2.0:1775768141.489776:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000100:00000001:0.0:1775768141.489781:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768141.489784:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:2.1:1775768141.489785:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 02000000:00000001:0.0:1775768141.489788:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:2.1:1775768141.489790:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 02000000:00000001:0.0:1775768141.489791:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768141.489796:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775768141.489800:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775768141.489800:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000008:00000001:0.0:1775768141.489804:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1775768141.489804:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000100:00000001:0.0:1775768141.489808:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768141.489812:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fd500. 00000400:00000200:2.0:1775768141.489812:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000040:0.0:1775768141.489817:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000400:00000200:2.0:1775768141.489822:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 02000000:00000001:0.0:1775768141.489823:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000200:2.0:1775768141.489829:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 02000000:00000001:0.0:1775768141.489832:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775768141.489832:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:0.0:1775768141.489836:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1775768141.489836:0:2399:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:0.0:1775768141.489837:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768141.489840:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768141.489845:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002b800. 00000100:00000001:0.0:1775768141.489850:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1775768141.489851:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fc000 x1862022294279040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768166 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768141.489856:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fd500 x1862022294279296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775768141.489869:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1775768141.489872:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:2.0:1775768141.489876:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fc000 x1862022294279040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768166 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768141.489877:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768141.489884:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f246bb2b] to pc [ptlrpcd_rcv+-1] req@ffff9942099fd500 x1862022294279296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768141.489890:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768141.489893:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768141.489900:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768141.489900:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768141.489902:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:2.0:1775768141.489908:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00080000:0.0:1775768141.489909:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775768141.489912:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1775768141.489913:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4300067645 00000100:00080000:0.0:1775768141.489914:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768141.489918:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768141.489921:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5393 00000100:00000010:2.0:1775768141.489921:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d63c0. 00000400:00000200:0.0:1775768141.489928:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768141.489930:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000400:00000200:0.0:1775768141.489932:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768141.489935:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1775768141.489936:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4300067645 00000100:00000040:0.0:1775768141.489938:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768141.489942:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000010:2.0:1775768141.489942:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d68a0. 00000100:00000040:0.0:1775768141.489946:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768141.489947:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775768141.489948:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768141.489952:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768141.489953:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:0.0:1775768141.489954:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768141.489956:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:0.0:1775768141.489958:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:0.0:1775768141.489960:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:2.0:1775768141.489962:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1775768141.489964:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768141.489967:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768141.489968:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775768141.489970:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000040:0.0:1775768141.489973:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768141.489975:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00100000:2.0:1775768141.489978:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768141/real 1775768141] req@ffff9942099fc000 x1862022294279040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1775768141.489979:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:0.0:1775768141.489985:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768141.489991:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1775768141.490003:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768141.490005:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:0.0:1775768141.490006:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:2.0:1775768141.490007:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 02000000:00000001:0.0:1775768141.490011:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1775768141.490011:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768141.490014:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 02000000:00000001:0.0:1775768141.490016:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768141.490018:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1775768141.490019:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775768141.490023:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000100:00000200:2.0:1775768141.490024:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fc000 x1862022294279040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:0.0:1775768141.490029:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768141.490032:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768141.490038:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ff100. 00000100:00000200:2.0:1775768141.490041:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fc000 x1862022294279040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1775768141.490042:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:0.0:1775768141.490046:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768141.490049:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768141.490053:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768141.490055:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1775768141.490055:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:0.0:1775768141.490059:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768141.490059:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768141.490064:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000010:0.0:1775768141.490066:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220028000. 00000100:00000001:2.0:1775768141.490067:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768141.490070:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768141.490074:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fc000 x1862022294279040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768141.490078:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ff100 x1862022294279424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775768141.490093:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768141.490097:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768141.490101:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768141.490102:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fc000 x1862022294279040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768166 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775768141.490108:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5e24158] to pc [ptlrpcd_rcv+-1] req@ffff9942099ff100 x1862022294279424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768141.490115:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768141.490120:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768141.490124:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768141.490124:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768141.490126:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1775768141.490129:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775768141.490130:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768141.490133:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768141.490134:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775768141.490137:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775768141.490138:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768141.490140:0:36:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768141.490144:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775768141.490145:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00080000:0.0:1775768141.490148:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5393 00000100:00000001:2.0:1775768141.490150:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768141.490177:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fc000 x1862022294279040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768166 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775768141.490184:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00100000:2.0:1775768141.490189:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294279040:192.168.202.156@tcp:250:kworker.0 00000400:00000200:0.0:1775768141.490191:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768141.490196:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1775768141.490198:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768141.490202:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768141.490202:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1775768141.490206:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768141.490209:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fc000 x1862022294279040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768166 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775768141.490210:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768141.490215:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768141.490220:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768141.490223:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000100:00000001:2.0:1775768141.490224:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1775768141.490225:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000001:2.0:1775768141.490227:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000040:0.0:1775768141.490228:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 02000000:00000010:2.0:1775768141.490230:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f800. 00000020:00000001:0.0:1775768141.490231:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:0.0:1775768141.490235:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 02000000:00000001:2.0:1775768141.490235:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768141.490238:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000040:0.0:1775768141.490240:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000040:2.0:1775768141.490241:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000100:00000001:0.0:1775768141.490244:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:2.0:1775768141.490245:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000040:0.0:1775768141.490247:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000010:2.0:1775768141.490249:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029400. 00000100:00000001:0.0:1775768141.490250:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768141.490253:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 02000000:00000001:2.0:1775768141.490255:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00080000:0.0:1775768141.490257:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 02000000:00000001:2.0:1775768141.490258:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768141.490261:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fc000. 00000100:00000001:0.0:1775768141.490262:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768141.490265:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1775768141.490266:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 02000000:00000001:0.0:1775768141.490269:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768141.490269:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1775768141.490274:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768141.490274:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768141.490278:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775768141.490281:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:0.0:1775768141.490283:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768141.490283:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768141.490286:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768141.490286:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768141.490290:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000010:0.0:1775768141.490292:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fc700. 00000100:00000001:2.0:1775768141.490292:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768141.490294:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000040:0.0:1775768141.490296:0:36:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00000100:00000040:2.0:1775768141.490300:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ff800 x1862022294279168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1775768141.490301:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768141.490303:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768141.490307:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768141.490309:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768141.490312:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768141.490314:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768141.490316:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768141.490317:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220029400. 02000000:00000001:2.0:1775768141.490319:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1775768141.490321:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768141.490322:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768141.490326:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768141.490329:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fc700 x1862022294279552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768141.490332:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294279168:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768141.490341:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768141.490343:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1775768141.490347:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1775768141.490347:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768141.490350:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1775768141.490351:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775768141.490354:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f800. 00000100:00000040:0.0:1775768141.490357:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003283b20a] to pc [ptlrpcd_rcv+-1] req@ffff9942099fc700 x1862022294279552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1775768141.490358:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768141.490366:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768141.490371:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000100:00000001:0.0:1775768141.490376:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775768141.490377:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768141.490382:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294279168, portal 10 00000100:00000040:0.0:1775768141.490383:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5404) 00000100:00000001:2.0:1775768141.490384:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768141.490387:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641210880 : -112966068340736 : ffff9942099ff800) 00000100:00000040:2.0:1775768141.490394:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ff800 x1862022294279168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768166 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768141.490400:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768141.490403:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768141.490406:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768141.490408:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294279168, offset 0 00000400:00000200:2.0:1775768141.490412:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768141.490416:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768141.490420:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768141.490424:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768141.490427:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768141.490430:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768141.490433:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768141.490435:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225354:225354:256:4294967295] 192.168.202.156@tcp LPNI seq info [225350:225350:8:4294967295] 00000400:00000200:2.0:1775768141.490439:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768141.490445:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768141.490449:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768141.490452:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768141.490456:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768141.490459:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775768141.490463:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768141.490465:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000100:2.0:1775768141.490469:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768141.490471:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768141.490473:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768141.490476:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768141.490479:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768141.490481:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768141.490483:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768141.490486:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ff800 x1862022294279168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768166 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768141.490494:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768141.490496:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ff800 x1862022294279168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768166 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768141.490515:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768141.490516:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768141.490519:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768141.490522:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:2.0:1775768141.490535:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768141.490537:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300066323 00000100:00000001:2.0:1775768141.490540:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300067645 00000100:00000001:2.0:1775768141.490542:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300066323 00000100:00000001:2.0:1775768141.490544:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300067645 00000100:00000001:2.0:1775768141.490546:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768141.490548:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768141.490549:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768141.490553:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768141/real 1775768141] req@ffff9942099ff800 x1862022294279168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768141.490560:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768141.490561:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768141.490564:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768141.490566:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768141.490567:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768141.490569:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768141.490572:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ff800 x1862022294279168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768141.490580:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ff800 x1862022294279168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768141.490586:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768141.490587:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768141.490589:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768141.490590:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768141.490593:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ff800 x1862022294279168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768141.490600:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768141.490603:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ff800 x1862022294279168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768166 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768141.490609:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768141.490612:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768141.490613:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768141.490614:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768141.490616:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768141.490618:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768141.490621:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768141.490624:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768141.490626:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768141.490629:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ff800 x1862022294279168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768166 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768141.490636:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294279168:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768141.490640:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768141.490643:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fd500 x1862022294279296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768141.490649:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768141.490650:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768141.490652:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768141.490654:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768141.490656:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768141.490658:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294279296:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768141.490661:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768141.490663:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768141.490665:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768141.490666:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768141.490669:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f000. 02000000:00000001:2.0:1775768141.490671:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768141.490674:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768141.490676:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768141.490679:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768141.490681:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294279296, portal 10 00000100:00000001:2.0:1775768141.490683:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768141.490684:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641201920 : -112966068349696 : ffff9942099fd500) 00000100:00000040:2.0:1775768141.490689:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fd500 x1862022294279296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768166 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768141.490695:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768141.490697:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768141.490699:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768141.490701:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294279296, offset 0 00000400:00000200:2.0:1775768141.490705:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768141.490708:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768141.490712:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768141.490714:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768141.490717:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768141.490719:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768141.490721:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768141.490724:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225355:225355:256:4294967295] 192.168.202.156@tcp LPNI seq info [225351:225351:8:4294967295] 00000400:00000200:2.0:1775768141.490728:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768141.490734:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768141.490737:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768141.490740:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768141.490743:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768141.490745:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775768141.490749:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768141.490751:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000100:2.0:1775768141.490754:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768141.490755:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768141.490757:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768141.490760:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768141.490763:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768141.490764:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768141.490766:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768141.490769:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fd500 x1862022294279296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768166 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768141.490775:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768141.490777:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fd500 x1862022294279296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768166 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768141.490783:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768141.490784:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768141.490786:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768141.490789:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:2.0:1775768141.490791:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768141.490793:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300066315 00000100:00000001:2.0:1775768141.490796:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300067645 00000100:00000001:2.0:1775768141.490798:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300066315 00000100:00000001:2.0:1775768141.490800:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300067645 00000100:00000001:2.0:1775768141.490802:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768141.490804:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768141.490806:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768141.490809:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768141/real 1775768141] req@ffff9942099fd500 x1862022294279296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768141.490816:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768141.490817:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768141.490819:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768141.490821:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768141.490822:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768141.490824:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768141.490827:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fd500 x1862022294279296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768141.490835:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fd500 x1862022294279296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768141.490841:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768141.490842:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768141.490844:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768141.490845:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768141.490848:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fd500 x1862022294279296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768141.490855:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768141.490858:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fd500 x1862022294279296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768166 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768141.490863:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768141.490865:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768141.490866:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768141.490867:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768141.490869:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768141.490871:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768141.490873:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768141.490876:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768141.490878:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768141.490882:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fd500 x1862022294279296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768166 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768141.490889:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294279296:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768141.490893:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768141.490896:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ff100 x1862022294279424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768141.490902:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768141.490903:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768141.490905:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768141.490907:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768141.490909:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768141.490912:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294279424:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768141.490915:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768141.490916:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768141.490918:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768141.490919:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768141.490922:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1e400. 02000000:00000001:2.0:1775768141.490924:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768141.490928:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768141.490930:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768141.490933:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768141.490935:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294279424, portal 4 00000100:00000001:2.0:1775768141.490937:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768141.490938:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641209088 : -112966068342528 : ffff9942099ff100) 00000100:00000040:2.0:1775768141.490942:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ff100 x1862022294279424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768166 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768141.490948:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768141.490950:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768141.490953:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768141.490955:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294279424, offset 0 00000400:00000200:2.0:1775768141.490959:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768141.490962:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768141.490966:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768141.490968:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768141.490970:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768141.490972:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768141.490974:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768141.490977:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225356:225356:256:4294967295] 192.168.202.156@tcp LPNI seq info [225352:225352:8:4294967295] 00000400:00000200:2.0:1775768141.490981:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768141.490987:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768141.490991:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768141.490993:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768141.490997:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768141.490999:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775768141.491003:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768141.491005:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000100:2.0:1775768141.491009:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768141.491010:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768141.491013:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768141.491016:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768141.491019:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768141.491020:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768141.491022:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768141.491025:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ff100 x1862022294279424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768166 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768141.491032:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768141.491035:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ff100 x1862022294279424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768166 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768141.491046:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768141.491047:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768141.491050:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768141.491052:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:2.0:1775768141.491056:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768141.491057:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300066338 00000100:00000001:2.0:1775768141.491060:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300067645 00000100:00000001:2.0:1775768141.491062:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300066338 00000100:00000001:2.0:1775768141.491063:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300067645 00000100:00000001:2.0:1775768141.491065:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768141.491067:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768141.491068:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768141.491080:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768141/real 1775768141] req@ffff9942099ff100 x1862022294279424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768141.491090:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768141.491091:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768141.491093:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768141.491095:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768141.491097:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768141.491099:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768141.491101:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ff100 x1862022294279424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768141.491108:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ff100 x1862022294279424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768141.491114:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768141.491116:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768141.491117:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768141.491118:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768141.491121:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ff100 x1862022294279424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768141.491129:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768141.491132:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ff100 x1862022294279424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768166 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768141.491139:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768141.491141:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768141.491143:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768141.491144:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768141.491145:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768141.491148:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768141.491150:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768141.491154:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768141.491179:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768141.491184:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ff100 x1862022294279424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768166 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768141.491192:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294279424:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768141.491196:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768141.491197:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768141.491200:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ff800 x1862022294279168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768166 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768141.491206:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768141.491208:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768141.491209:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f800. 02000000:00000001:2.0:1775768141.491212:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768141.491214:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768141.491215:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:2.0:1775768141.491217:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768141.491218:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002bc00. 02000000:00000001:2.0:1775768141.491221:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768141.491223:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768141.491224:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ff800. 00000100:00000001:2.0:1775768141.491226:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768141.491228:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768141.491229:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768141.491232:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fd500 x1862022294279296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768166 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768141.491239:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768141.491240:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768141.491241:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f000. 02000000:00000001:2.0:1775768141.491243:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768141.491244:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768141.491245:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:2.0:1775768141.491246:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768141.491248:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002b800. 02000000:00000001:2.0:1775768141.491250:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768141.491251:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768141.491252:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fd500. 00000100:00000001:2.0:1775768141.491254:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768141.491255:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768141.491256:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768141.491260:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ff100 x1862022294279424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768166 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768141.491266:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768141.491267:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768141.491268:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1e400. 02000000:00000001:2.0:1775768141.491271:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768141.491272:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768141.491273:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:2.0:1775768141.491275:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768141.491276:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028000. 02000000:00000001:2.0:1775768141.491278:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768141.491279:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768141.491280:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ff100. 00000100:00000001:2.0:1775768141.491283:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768141.491284:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768141.491286:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768141.491289:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768141.491291:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768141.491292:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768141.491294:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768141.491295:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768141.491299:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fc700 x1862022294279552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768141.491305:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768141.491306:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768141.491308:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768141.491309:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768141.491311:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768141.491314:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294279552:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768141.491317:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768141.491319:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768141.491320:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768141.491322:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768141.491325:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1e400. 02000000:00000001:2.0:1775768141.491327:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768141.491330:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768141.491332:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768141.491335:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768141.491338:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294279552, portal 4 00000100:00000001:2.0:1775768141.491340:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768141.491341:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641198336 : -112966068353280 : ffff9942099fc700) 00000100:00000040:2.0:1775768141.491345:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fc700 x1862022294279552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768166 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768141.491351:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768141.491353:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768141.491355:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768141.491358:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294279552, offset 0 00000400:00000200:2.0:1775768141.491361:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768141.491365:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768141.491369:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768141.491371:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768141.491374:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768141.491376:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768141.491378:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768141.491381:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225357:225357:256:4294967295] 192.168.202.156@tcp LPNI seq info [225353:225353:8:4294967295] 00000400:00000200:2.0:1775768141.491385:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768141.491391:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768141.491395:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768141.491399:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768141.491403:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768141.491405:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775768141.491409:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768141.491412:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000100:2.0:1775768141.491415:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768141.491417:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768141.491419:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768141.491423:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768141.491426:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768141.491427:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768141.491429:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768141.491432:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fc700 x1862022294279552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768166 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768141.491440:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768141.491442:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fc700 x1862022294279552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768166 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768141.491448:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768141.491449:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768141.491452:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768141.491454:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:2.0:1775768141.491457:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768141.491459:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300066342 00000100:00000001:2.0:1775768141.491462:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300067645 00000100:00000001:2.0:1775768141.491463:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300066342 00000100:00000001:2.0:1775768141.491465:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300067645 00000100:00000001:2.0:1775768141.491467:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768141.491469:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768141.491471:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768141.491474:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768141/real 1775768141] req@ffff9942099fc700 x1862022294279552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768141.491481:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768141.491482:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768141.491484:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768141.491487:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768141.491488:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768141.491491:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768141.491494:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fc700 x1862022294279552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768141.491501:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fc700 x1862022294279552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768141.491507:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768141.491509:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768141.491510:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768141.491512:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768141.491515:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fc700 x1862022294279552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768141.491521:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768141.491539:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fc700 x1862022294279552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768166 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768141.491545:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768141.491548:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768141.491550:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768141.491551:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768141.491552:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768141.491555:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768141.491557:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768141.491560:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768141.491563:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768141.491566:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fc700 x1862022294279552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768166 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768141.491574:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294279552:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768141.491577:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768141.491579:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768141.491581:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fc700 x1862022294279552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768166 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768141.491588:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768141.491589:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768141.491590:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1e400. 02000000:00000001:2.0:1775768141.491593:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768141.491594:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768141.491595:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:2.0:1775768141.491597:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768141.491599:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029400. 02000000:00000001:2.0:1775768141.491601:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768141.491602:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768141.491603:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fc700. 00000100:00000001:2.0:1775768141.491605:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768141.491606:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768141.491608:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768141.491611:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768141.491612:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768141.491614:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768141.491616:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768141.553246:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768141.553254:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768141.553256:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768141.553259:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768141.553264:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768141.553267:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768141.553269:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768141.553271:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768141.553275:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768141.553276:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768141.553279:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768141.553280:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768141.553283:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768141.553284:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768141.553287:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768141.553288:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768141.553345:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768141.553348:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768141.553351:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768141.617252:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768141.617257:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768141.617260:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768141.617264:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768141.617268:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768141.617272:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768141.617274:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768141.617276:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768141.617279:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768142.513282:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768142.513291:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768142.513293:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768142.513299:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768142.513302:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768142.513306:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768142.513309:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768142.513312:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768142.513315:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768142.513453:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768142.513458:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768142.513460:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768142.513462:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768142.513465:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768142.513466:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768142.513467:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768142.513468:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768142.513470:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768142.513492:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768142.513497:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768142.513500:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768142.513505:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768142.513508:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768142.513512:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768142.513528:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768142.513531:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768142.513535:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768142.577266:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768142.577272:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768142.577276:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768142.577277:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768142.577280:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768142.577281:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768142.577283:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768142.577284:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768142.577287:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768142.577287:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768142.577335:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768142.577338:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768142.577341:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768142.641274:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768142.641282:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768142.641284:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768142.641290:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768142.641294:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768142.641299:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768142.641302:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768142.641304:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768142.641307:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768142.833230:0:14953:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4300067645, at 4300067648 00000100:00000040:1.0:1775768142.833230:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4300067645, at 4300067648 00000100:00000010:0.0:1775768142.833245:0:14953:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d63c0. 00000100:00000010:1.0:1775768142.833247:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d68a0. 00000100:00000001:1.0:1775768143.537244:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768143.537245:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768143.537251:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768143.537251:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768143.537253:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768143.537254:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768143.537261:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:2.0:1775768143.537261:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768143.537267:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768143.537267:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768143.537272:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768143.537272:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768143.537276:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768143.537279:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768143.537282:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768143.537283:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768143.537286:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768143.537289:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768143.537382:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768143.537386:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768143.537387:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768143.537390:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768143.537393:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768143.537395:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768143.537397:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768143.537399:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768143.537401:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768143.601264:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768143.601269:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768143.601274:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768143.601276:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768143.601279:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768143.601281:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768143.601284:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768143.601285:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768143.601287:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768143.601289:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768143.601339:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768143.601342:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768143.601346:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768143.665230:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768143.665236:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768143.665238:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768143.665242:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768143.665245:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768143.665249:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768143.665251:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768143.665253:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768143.665255:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768144.561236:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768144.561243:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768144.561245:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768144.561251:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768144.561254:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768144.561258:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768144.561260:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768144.561262:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768144.561265:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768144.561345:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768144.561348:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768144.561349:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768144.561352:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768144.561354:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768144.561356:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768144.561358:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768144.561359:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768144.561361:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768144.561434:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768144.561437:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768144.561438:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768144.561441:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768144.561443:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768144.561445:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768144.561446:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768144.561448:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768144.561449:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768144.625250:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768144.625255:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768144.625259:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768144.625260:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768144.625262:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768144.625263:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768144.625266:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768144.625266:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768144.625268:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768144.625269:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768144.625311:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768144.625315:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768144.625321:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768144.689220:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768144.689225:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768144.689226:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768144.689229:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768144.689232:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768144.689235:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768144.689237:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768144.689238:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768144.689240:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768145.585233:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768145.585233:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768145.585240:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768145.585240:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768145.585244:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768145.585244:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768145.585245:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768145.585249:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:1.0:1775768145.585251:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775768145.585251:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768145.585252:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768145.585258:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:2.0:1775768145.585258:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775768145.585258:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768145.585262:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768145.585263:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768145.585263:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768145.585265:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768145.585270:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768145.585272:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768145.585275:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768145.585276:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768145.585278:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768145.585281:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768145.585282:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768145.585284:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768145.585287:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768145.649257:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768145.649261:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768145.649266:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768145.649267:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768145.649270:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768145.649271:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768145.649274:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768145.649275:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768145.649278:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768145.649280:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768145.649333:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768145.649336:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768145.649339:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768145.713254:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768145.713260:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768145.713262:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768145.713267:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768145.713271:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768145.713274:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768145.713276:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768145.713278:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768145.713281:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768146.609249:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768146.609249:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768146.609249:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768146.609255:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768146.609255:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768146.609255:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768146.609257:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768146.609258:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768146.609259:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1775768146.609261:0:13471:0:(rw.c:1481:ll_writepages()) Process entered 00000100:00100000:2.0:1775768146.609262:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775768146.609263:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:1.0:1775768146.609264:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000080:00000001:0.0:1775768146.609265:0:13471:0:(file.c:5250:cl_sync_file_range()) Process entered 00000100:00000001:2.0:1775768146.609268:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768146.609268:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1775768146.609269:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00000001:2.0:1775768146.609274:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768146.609274:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:0.0:1775768146.609278:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff9942103bed80. 00000100:00000001:3.0:1775768146.609279:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768146.609280:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768146.609281:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768146.609283:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:0.0:1775768146.609284:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0f600. 00000100:00000001:3.0:1775768146.609284:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768146.609285:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768146.609288:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:0.0:1775768146.609290:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210b0fc00. 00000100:00000001:1.0:1775768146.609291:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768146.609291:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768146.609294:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:0.0:1775768146.609295:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942078a3d80. 00000100:00000001:1.0:1775768146.609298:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:0.0:1775768146.609301:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff9942086e6468. 00020000:00000010:0.0:1775768146.609305:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994209592f00. 00008000:00000010:0.0:1775768146.609309:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff99422035b880. 00000080:00000010:0.0:1775768146.609313:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994207e3e150. 00000080:00000010:0.0:1775768146.609316:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b2238. 00000020:00000010:0.0:1775768146.609319:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0ec00. 00000008:00000010:0.0:1775768146.609322:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff99420906be40. 00000100:00080000:3.0:1775768146.609324:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00020000:00000010:0.0:1775768146.609326:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff9942103ccfc0. 00000100:00080000:3.0:1775768146.609329:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00008000:00000010:0.0:1775768146.609331:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076afa0. 00000100:00000001:3.0:1775768146.609332:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000080:00000010:0.0:1775768146.609336:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942107690f0. 00000100:00080000:3.0:1775768146.609336:0:36:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775768146.609339:0:36:0:(import.c:484:import_select_connection()) Process entered 00000020:00000001:0.0:1775768146.609340:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107752095112 : -112965957456504 : ffff9942103bed88) 00000020:00001000:0.0:1775768146.609342:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff9942103bed80 00000100:00080000:3.0:1775768146.609344:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5399 00000400:00000200:3.0:1775768146.609351:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:0.0:1775768146.609354:0:13471:0:(cl_object.c:151:cl_object_top()) ffff99420791a5f8 -> ffff99420791a5f8 00000020:00000001:0.0:1775768146.609356:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000400:00000200:3.0:1775768146.609357:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000080:00000001:0.0:1775768146.609358:0:13471:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:0.0:1775768146.609361:0:13471:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x10176:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000100:00080000:3.0:1775768146.609361:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000020:00000001:0.0:1775768146.609366:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000100:00000040:3.0:1775768146.609366:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000001:0.0:1775768146.609369:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00000001:3.0:1775768146.609370:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000080:00000001:0.0:1775768146.609371:0:13471:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768146.609373:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00020000:00000002:0.0:1775768146.609375:0:13471:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x10176:0x0]io ffff9942077b2298 type 6 ignore/verify layout 1/0 00000100:00000001:3.0:1775768146.609376:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00020000:00000002:0.0:1775768146.609381:0:13471:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207dfb6c8) owner 0000000000000000/ffff994220080000 00000020:00000001:3.0:1775768146.609381:0:36:0:(genops.c:811:class_conn2export()) Process entered 00020000:00000001:0.0:1775768146.609383:0:13471:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00000020:00000040:3.0:1775768146.609384:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00020000:00000001:0.0:1775768146.609386:0:13471:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00000020:00000001:3.0:1775768146.609386:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00020000:00000001:0.0:1775768146.609388:0:13471:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00000020:00000040:3.0:1775768146.609389:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00020000:00000001:0.0:1775768146.609390:0:13471:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768146.609392:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00020000:00000001:0.0:1775768146.609393:0:13471:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1775768146.609394:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00020000:00000001:0.0:1775768146.609395:0:13471:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1775768146.609397:0:13471:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768146.609399:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000100:00000040:3.0:1775768146.609399:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768146.609401:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:0.0:1775768146.609402:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1775768146.609404:0:13471:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00020000:00000002:0.0:1775768146.609406:0:13471:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207dfb6c8) owner 0000000000000000/ffff994220080000 00000100:00000040:3.0:1775768146.609406:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000001:0.0:1775768146.609409:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768146.609409:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768146.609414:0:13471:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000040:3.0:1775768146.609414:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000020:00000001:0.0:1775768146.609415:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00020000:00000001:0.0:1775768146.609417:0:13471:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00000100:00080000:3.0:1775768146.609417:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00020000:00200000:0.0:1775768146.609419:0:13471:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00000001:3.0:1775768146.609420:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1775768146.609423:0:13471:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 02000000:00000001:3.0:1775768146.609423:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775768146.609425:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1775768146.609426:0:13471:0:(lov_io.c:155:lov_sub_get()) Process entered 00020000:00000001:0.0:1775768146.609428:0:13471:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000020:00000001:0.0:1775768146.609429:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 02000000:00000001:3.0:1775768146.609433:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1775768146.609434:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff9942103be000. 00000100:00000001:3.0:1775768146.609435:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000020:00000010:0.0:1775768146.609437:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0ea00. 00000100:00000001:3.0:1775768146.609438:0:36:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1775768146.609439:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210b0fa00. 00000020:00000010:0.0:1775768146.609442:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942078a30c0. 00000100:00000010:3.0:1775768146.609444:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aad80. 00000008:00000010:0.0:1775768146.609445:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff9942086e4e18. 00000020:00000040:3.0:1775768146.609447:0:36:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 00020000:00000010:0.0:1775768146.609448:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994209592e40. 00008000:00000010:0.0:1775768146.609452:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994220358000. 02000000:00000001:3.0:1775768146.609452:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000080:00000010:0.0:1775768146.609454:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994207e3e3f0. 02000000:00000001:3.0:1775768146.609454:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1775768146.609458:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b0ff8. 00000100:00000001:3.0:1775768146.609459:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000020:00000010:0.0:1775768146.609461:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0e000. 02000000:00000001:3.0:1775768146.609461:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000008:00000010:0.0:1775768146.609465:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff99420906a6e0. 02000000:00000001:3.0:1775768146.609466:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:0.0:1775768146.609467:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff9942103cc480. 00008000:00000010:0.0:1775768146.609469:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076adb0. 00000080:00000010:0.0:1775768146.609473:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff994210769870. 02000000:00000010:3.0:1775768146.609473:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2000. 00000020:00000001:0.0:1775768146.609484:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107752091656 : -112965957459960 : ffff9942103be008) 00000020:00001000:0.0:1775768146.609487:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff9942103be000 00000020:00000001:0.0:1775768146.609488:0:13471:0:(cl_object.c:151:cl_object_top()) ffff99420791ae90 -> ffff99420791a5f8 00000020:00000001:0.0:1775768146.609490:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:0.0:1775768146.609492:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1775768146.609493:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:0.0:1775768146.609495:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768146.609495:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768146.609498:0:13471:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768146.609500:0:13471:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107752153128 : -112965957398488 : ffff9942103cd028) 00000020:00000001:0.0:1775768146.609502:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000020:00000001:0.0:1775768146.609503:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768146.609505:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aad80 x1862022294279680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00020000:00000001:0.0:1775768146.609506:0:13471:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768146.609509:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768146.609512:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00020000:00000001:0.0:1775768146.609514:0:13471:0:(lov_io.c:1108:lov_io_lock()) Process entered 00020000:00000001:0.0:1775768146.609516:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:0.0:1775768146.609518:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:0.0:1775768146.609519:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000100:00000001:3.0:1775768146.609520:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000020:00000001:0.0:1775768146.609521:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768146.609522:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768146.609523:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768146.609525:0:13471:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768146.609525:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768146.609529:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:0.0:1775768146.609530:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768146.609531:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768146.609532:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000045ccf08] to pc [ptlrpcd_rcv+-1] req@ffff9942099aad80 x1862022294279680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775768146.609533:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00020000:00000001:0.0:1775768146.609534:0:13471:0:(lov_io.c:1114:lov_io_start()) Process entered 00020000:00000001:0.0:1775768146.609535:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:0.0:1775768146.609537:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00000008:00000001:0.0:1775768146.609539:0:13471:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000008:00000001:0.0:1775768146.609542:0:13471:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000008:00000001:0.0:1775768146.609547:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:0.0:1775768146.609549:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:0.0:1775768146.609550:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1775768146.609552:0:13471:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000008:00000001:0.0:1775768146.609552:0:13471:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000008:00000001:0.0:1775768146.609554:0:13471:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107754940544 : -112965954611072 : ffff994210675880) 00000100:00000001:3.0:1775768146.609554:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000008:00000020:0.0:1775768146.609556:0:13471:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994210675880 ready 0|-|- wr 1|-|+ rd 0|- 10 in flight 00000100:00080000:3.0:1775768146.609557:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000008:00000001:0.0:1775768146.609560:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000100:00080000:3.0:1775768146.609561:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000008:00000020:0.0:1775768146.609562:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:0.0:1775768146.609564:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768146.609564:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000008:00000001:0.0:1775768146.609566:0:13471:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000100:00080000:3.0:1775768146.609566:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000008:00000020:0.0:1775768146.609570:0:13471:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994218232fc0 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000100:00000001:3.0:1775768146.609570:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775768146.609574:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5399 00000008:00000001:0.0:1775768146.609575:0:13471:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1775768146.609578:0:13471:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768146.609578:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768146.609581:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1775768146.609582:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775768146.609582:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768146.609584:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00020000:00000001:0.0:1775768146.609586:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775768146.609588:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00020000:00000001:0.0:1775768146.609589:0:13471:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768146.609591:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000020:00000001:0.0:1775768146.609592:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768146.609592:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099aad80 x1862022294279680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775768146.609596:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00020000:00000001:0.0:1775768146.609598:0:13471:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00000100:00000040:3.0:1775768146.609598:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00020000:00000001:0.0:1775768146.609601:0:13471:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000100:00000001:3.0:1775768146.609602:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:0.0:1775768146.609603:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000001:0.0:1775768146.609605:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00020000:00000001:0.0:1775768146.609607:0:13471:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768146.609607:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00020000:00000001:0.0:1775768146.609611:0:13471:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000100:00000001:3.0:1775768146.609612:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768146.609614:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00000100:00000001:2.0:1775768146.609615:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000001:3.0:1775768146.609616:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000001:0.0:1775768146.609617:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000100:00000001:2.0:1775768146.609617:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775768146.609619:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00020000:00000001:0.0:1775768146.609620:0:13471:0:(lov_io.c:1208:lov_io_unlock()) Process entered 02000000:00000001:2.0:1775768146.609621:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:3.0:1775768146.609621:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00020000:00000001:0.0:1775768146.609622:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000040:3.0:1775768146.609624:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:0.0:1775768146.609625:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 02000000:00000001:2.0:1775768146.609625:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1775768146.609627:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:0.0:1775768146.609628:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 02000000:00000001:2.0:1775768146.609628:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768146.609629:0:13471:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768146.609631:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00020000:00000001:0.0:1775768146.609633:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768146.609636:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099aad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294279680:192.168.202.156@tcp:250:kworker.0 00000100:00000040:3.0:1775768146.609636:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00020000:00000001:0.0:1775768146.609637:0:13471:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000020:00000001:0.0:1775768146.609639:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000100:00000001:2.0:1775768146.609641:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000001:0.0:1775768146.609642:0:13471:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000020:00000001:0.0:1775768146.609645:0:13471:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 02000000:00000001:2.0:1775768146.609645:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1775768146.609645:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:0.0:1775768146.609648:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000100:00000040:3.0:1775768146.609648:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000001:2.0:1775768146.609649:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768146.609650:0:13471:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00020000:00000001:0.0:1775768146.609652:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 02000000:00000001:2.0:1775768146.609653:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1775768146.609653:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768146.609654:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000001:0.0:1775768146.609657:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000020:00000040:3.0:1775768146.609657:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00020000:00000001:0.0:1775768146.609658:0:13471:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775768146.609659:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1e400. 00020000:00000001:0.0:1775768146.609662:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768146.609662:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 02000000:00000001:2.0:1775768146.609663:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768146.609667:0:13471:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000020:00000001:0.0:1775768146.609668:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000100:00000001:3.0:1775768146.609668:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768146.609670:0:13471:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768146.609672:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:2.0:1775768146.609674:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:0.0:1775768146.609675:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1775768146.609677:0:13471:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:0.0:1775768146.609678:0:13471:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000400:00000010:2.0:1775768146.609678:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 02000000:00000001:3.0:1775768146.609678:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1775768146.609681:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:0.0:1775768146.609683:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 02000000:00000001:3.0:1775768146.609683:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1775768146.609686:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff9942103be000 00000400:00000010:2.0:1775768146.609686:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000001:3.0:1775768146.609686:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000020:00000010:0.0:1775768146.609690:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210b0fa00. 00000008:00100000:3.0:1775768146.609691:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00000200:2.0:1775768146.609692:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294279680, portal 25 00000008:00000001:3.0:1775768146.609694:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1775768146.609695:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942078a30c0. 00000100:00000001:2.0:1775768146.609696:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775768146.609696:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:0.0:1775768146.609699:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff9942086e4e18. 00000100:00000001:2.0:1775768146.609699:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640864128 : -112966068687488 : ffff9942099aad80) 00000100:00000010:3.0:1775768146.609701:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aa680. 00020000:00000010:0.0:1775768146.609704:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994209592e40. 00000020:00000040:3.0:1775768146.609704:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00008000:00000010:0.0:1775768146.609708:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994220358000. 02000000:00000001:3.0:1775768146.609708:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775768146.609710:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1775768146.609712:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994207e3e3f0. 00000100:00000001:3.0:1775768146.609714:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000040:2.0:1775768146.609715:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099aad80 x1862022294279680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768171 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000010:0.0:1775768146.609716:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b0ff8. 02000000:00000001:3.0:1775768146.609716:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775768146.609718:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1775768146.609721:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0ea00. 02000000:00000010:3.0:1775768146.609723:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0c00. 00000008:00000010:0.0:1775768146.609725:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff99420906a6e0. 00000100:00000001:3.0:1775768146.609727:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:0.0:1775768146.609728:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff9942103cc480. 00008000:00000010:0.0:1775768146.609733:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076adb0. 00000100:00100000:3.0:1775768146.609735:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aa680 x1862022294279808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000080:00000010:0.0:1775768146.609738:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff994210769870. 00000100:00000001:2.0:1775768146.609739:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000010:0.0:1775768146.609742:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0e000. 00000020:00000010:0.0:1775768146.609746:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff9942103be000. 00000100:00000040:2.0:1775768146.609747:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000100:00000001:3.0:1775768146.609749:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00020000:00000001:0.0:1775768146.609750:0:13471:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:0.0:1775768146.609752:0:13471:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000100:00000001:3.0:1775768146.609752:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775768146.609753:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000080:00000001:0.0:1775768146.609754:0:13471:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000080:00200000:0.0:1775768146.609757:0:13471:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x10176:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00000040:3.0:1775768146.609757:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006cb1597d] to pc [ptlrpcd_rcv+-1] req@ffff9942099aa680 x1862022294279808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768146.609759:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294279680, offset 0 00000080:00000001:0.0:1775768146.609761:0:13471:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1775768146.609762:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1775768146.609766:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff9942103bed80 00000020:00000010:0.0:1775768146.609769:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210b0fc00. 00000100:00000001:3.0:1775768146.609772:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1775768146.609773:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942078a3d80. 00000400:00000200:2.0:1775768146.609773:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00080000:3.0:1775768146.609775:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000008:00000010:0.0:1775768146.609776:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff9942086e6468. 00020000:00000010:0.0:1775768146.609781:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994209592f00. 00000100:00080000:3.0:1775768146.609781:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775768146.609784:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00008000:00000010:0.0:1775768146.609785:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff99422035b880. 00000400:00000200:2.0:1775768146.609785:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00080000:3.0:1775768146.609786:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000080:00000010:0.0:1775768146.609789:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994207e3e150. 00000100:00000001:3.0:1775768146.609790:0:36:0:(import.c:484:import_select_connection()) Process entered 00000080:00000010:0.0:1775768146.609792:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b2238. 00000400:00000200:2.0:1775768146.609795:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00080000:3.0:1775768146.609795:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5399 00000020:00000010:0.0:1775768146.609797:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0f600. 00000400:00000200:2.0:1775768146.609802:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768146.609807:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000008:00000010:0.0:1775768146.609808:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff99420906be40. 00000400:00000200:2.0:1775768146.609810:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00020000:00000010:0.0:1775768146.609814:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff9942103ccfc0. 00000400:00000200:2.0:1775768146.609814:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00008000:00000010:0.0:1775768146.609819:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076afa0. 00000400:00000200:2.0:1775768146.609819:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225358:225358:256:4294967295] 192.168.202.156@tcp LPNI seq info [225354:225354:8:4294967295] 00000080:00000010:0.0:1775768146.609824:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942107690f0. 00000400:00000200:2.0:1775768146.609825:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000010:0.0:1775768146.609830:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0ec00. 00000400:00000200:3.0:1775768146.609830:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000010:0.0:1775768146.609836:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff9942103bed80. 00000400:00000200:3.0:1775768146.609839:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000400:00000200:2.0:1775768146.609840:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000080:00000001:0.0:1775768146.609841:0:13471:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775768146.609844:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000080:00000001:0.0:1775768146.609847:0:13471:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1775768146.609851:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00000040:3.0:1775768146.609852:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000800:00000010:2.0:1775768146.609855:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000100:00000001:3.0:1775768146.609856:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775768146.609861:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000800:00000200:2.0:1775768146.609865:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:3.0:1775768146.609865:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000800:00000200:2.0:1775768146.609870:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000020:00000001:3.0:1775768146.609870:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775768146.609873:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:3.0:1775768146.609877:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000800:00000200:2.1:1775768146.609877:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000040:3.0:1775768146.609881:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000800:00000200:2.1:1775768146.609881:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000020:00000001:3.0:1775768146.609886:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000800:00000100:2.0:1775768146.609888:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000020:00000001:3.0:1775768146.609891:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000800:00000010:2.0:1775768146.609892:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768146.609896:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:3.0:1775768146.609898:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000200:2.0:1775768146.609902:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000100:00000001:3.0:1775768146.609903:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1775768146.609905:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768146.609908:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000040:3.0:1775768146.609909:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768146.609911:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1775768146.609912:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000200:2.0:1775768146.609916:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099aad80 x1862022294279680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768171 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1775768146.609917:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:3.0:1775768146.609924:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775768146.609929:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1775768146.609929:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768146.609933:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099aad80 x1862022294279680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768171 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1775768146.609933:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775768146.609937:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768146.609940:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768146.609941:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768146.609942:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775768146.609946:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775768146.609948:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1775768146.609948:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775768146.609953:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768146.609956:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775768146.609957:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768146.609958:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4300072765 00000100:00000010:2.0:1775768146.609963:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d6a80. 00000100:00000010:3.0:1775768146.609963:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ab800. 00000020:00000040:3.0:1775768146.609968:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000100:00000001:2.0:1775768146.609970:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 02000000:00000001:3.0:1775768146.609972:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775768146.609974:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768146.609975:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4300072765 00000100:00000001:3.0:1775768146.609978:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000010:2.0:1775768146.609979:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d6e40. 02000000:00000001:3.0:1775768146.609981:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1775768146.609982:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 02000000:00000001:3.0:1775768146.609984:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768146.609985:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768146.609989:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768146.609990:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 02000000:00000010:3.0:1775768146.609991:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b1000. 00000100:00100000:2.0:1775768146.609994:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768146/real 1775768146] req@ffff9942099aad80 x1862022294279680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768146.609997:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1775768146.610004:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768146.610006:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00100000:3.0:1775768146.610007:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ab800 x1862022294279936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:2.0:1775768146.610008:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768146.610011:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768146.610014:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768146.610017:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1775768146.610023:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775768146.610025:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1775768146.610030:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099aad80 x1862022294279680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775768146.610032:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000089df80c0] to pc [ptlrpcd_rcv+-1] req@ffff9942099ab800 x1862022294279936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768146.610050:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1775768146.610051:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099aad80 x1862022294279680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768146.610052:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768146.610059:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775768146.610061:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1775768146.610064:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00080000:3.0:1775768146.610064:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775768146.610067:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768146.610067:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775768146.610070:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:3.0:1775768146.610071:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5399 00000100:00000001:2.0:1775768146.610072:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768146.610078:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099aad80 x1862022294279680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1775768146.610078:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775768146.610084:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775768146.610088:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1775768146.610089:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775768146.610091:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768146.610094:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768146.610095:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099aad80 x1862022294279680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768171 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775768146.610097:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768146.610102:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775768146.610106:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775768146.610108:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000100:00000001:2.0:1775768146.610109:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768146.610112:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1775768146.610112:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:2.0:1775768146.610116:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768146.610116:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1775768146.610116:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000100:00000001:2.0:1775768146.610119:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768146.610121:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:3.0:1775768146.610121:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00080000:2.0:1775768146.610126:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000020:00000001:3.0:1775768146.610126:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00080000:2.0:1775768146.610131:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768146.610134:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775768146.610135:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768146.610139:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768146.610142:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099aad80 x1862022294279680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768171 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775768146.610145:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768146.610149:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775768146.610173:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:3.0:1775768146.610177:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775768146.610181:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768146.610184:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:2.0:1775768146.610186:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099aad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294279680:192.168.202.156@tcp:250:kworker.0 02000000:00000001:3.0:1775768146.610188:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775768146.610193:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768146.610195:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775768146.610196:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:3.0:1775768146.610197:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000100:00000001:2.0:1775768146.610199:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000008:00000001:3.0:1775768146.610201:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768146.610204:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099aad80 x1862022294279680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768171 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768146.610204:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775768146.610210:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aa300. 00000100:00000001:2.0:1775768146.610211:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768146.610214:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000040:3.0:1775768146.610214:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000010:2.0:1775768146.610217:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1e400. 02000000:00000001:3.0:1775768146.610218:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775768146.610220:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:3.0:1775768146.610221:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768146.610223:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768146.610224:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000100:00000001:3.0:1775768146.610224:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000020:00000001:2.0:1775768146.610228:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000001:3.0:1775768146.610228:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000010:2.0:1775768146.610231:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b2000. 02000000:00000001:3.0:1775768146.610232:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768146.610234:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768146.610236:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768146.610237:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099aad80. 02000000:00000010:3.0:1775768146.610238:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2000. 00000100:00000001:2.0:1775768146.610240:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768146.610242:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768146.610243:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768146.610244:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768146.610249:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768146.610251:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768146.610252:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aa300 x1862022294280064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775768146.610253:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768146.610255:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768146.610256:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768146.610261:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099aa680 x1862022294279808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768146.610269:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775768146.610270:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768146.610272:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768146.610273:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768146.610274:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768146.610276:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768146.610279:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768146.610281:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f33bdf15] to pc [ptlrpcd_rcv+-1] req@ffff9942099aa300 x1862022294280064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768146.610283:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099aa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294279808:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768146.610289:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768146.610291:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768146.610292:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768146.610296:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768146.610299:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1e400. 00000100:00000001:3.0:1775768146.610299:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768146.610303:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768146.610304:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768146.610308:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000400:00000200:2.0:1775768146.610309:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775768146.610310:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000400:00000010:2.0:1775768146.610312:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000100:00080000:3.0:1775768146.610312:0:36:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000010:2.0:1775768146.610315:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000001:3.0:1775768146.610316:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00000200:2.0:1775768146.610318:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294279808, portal 10 00000100:00000001:2.0:1775768146.610321:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1775768146.610321:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5399 00000100:00000001:2.0:1775768146.610323:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640862336 : -112966068689280 : ffff9942099aa680) 00000400:00000200:3.0:1775768146.610328:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000040:2.0:1775768146.610330:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099aa680 x1862022294279808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768171 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1775768146.610335:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:2.0:1775768146.610339:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1775768146.610340:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775768146.610343:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000100:00000040:3.0:1775768146.610347:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768146.610350:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:2.0:1775768146.610352:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000040:3.0:1775768146.610352:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768146.610355:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000200:2.0:1775768146.610356:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294279808, offset 0 00000020:00000001:3.0:1775768146.610360:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775768146.610363:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000400:00000200:2.0:1775768146.610364:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000020:00000001:3.0:1775768146.610367:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:2.0:1775768146.610368:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000040:3.0:1775768146.610370:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000400:00000200:2.0:1775768146.610373:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000020:00000001:3.0:1775768146.610374:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000400:00000200:2.0:1775768146.610376:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:3.0:1775768146.610379:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000400:00000200:2.0:1775768146.610380:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000100:00000040:3.0:1775768146.610385:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000200:2.0:1775768146.610386:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1775768146.610389:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1775768146.610390:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00000040:3.0:1775768146.610394:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000200:2.0:1775768146.610396:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225359:225359:256:4294967295] 192.168.202.156@tcp LPNI seq info [225355:225355:8:4294967295] 00000100:00000001:3.0:1775768146.610397:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775768146.610401:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000400:00000200:2.0:1775768146.610404:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00080000:3.0:1775768146.610406:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775768146.610409:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768146.610413:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:2.0:1775768146.610415:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 02000000:00000001:3.0:1775768146.610416:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:2.0:1775768146.610421:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 02000000:00000001:3.0:1775768146.610421:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768146.610423:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775768146.610425:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000800:00000010:2.0:1775768146.610426:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000008:00000001:3.0:1775768146.610427:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768146.610430:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1775768146.610434:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000010:3.0:1775768146.610435:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8a80. 00000800:00000200:2.0:1775768146.610437:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000020:00000040:3.0:1775768146.610438:0:36:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:3.0:1775768146.610441:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775768146.610443:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768146.610447:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000800:00000200:2.1:1775768146.610447:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 02000000:00000001:3.0:1775768146.610450:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:2.1:1775768146.610451:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 02000000:00000001:3.0:1775768146.610453:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1775768146.610458:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768146.610460:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 02000000:00000010:3.0:1775768146.610461:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b3400. 00000400:00000200:2.0:1775768146.610465:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:3.0:1775768146.610465:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768146.610471:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000100:00100000:3.0:1775768146.610474:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8a80 x1862022294280192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1775768146.610488:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768146.610490:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768146.610494:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768146.610499:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099aa680 x1862022294279808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768171 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768146.610501:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775768146.610506:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768146.610510:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768146.610515:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099aa680 x1862022294279808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768171 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775768146.610515:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006c3265af] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8a80 x1862022294280192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768146.610530:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768146.610534:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775768146.610536:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768146.610537:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768146.610540:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5409) 00000100:00000040:2.0:1775768146.610541:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:2.0:1775768146.610545:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768146.610546:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300071443 00000100:00000001:2.0:1775768146.610548:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300072765 00000100:00000001:2.0:1775768146.610550:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300071443 00000100:00000001:2.0:1775768146.610552:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300072765 00000100:00000001:2.0:1775768146.610553:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768146.610555:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768146.610556:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768146.610558:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768146/real 1775768146] req@ffff9942099aa680 x1862022294279808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768146.610562:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768146.610563:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768146.610564:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768146.610565:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768146.610566:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768146.610567:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768146.610569:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099aa680 x1862022294279808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768146.610573:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099aa680 x1862022294279808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768146.610577:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768146.610577:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768146.610578:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768146.610579:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768146.610581:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099aa680 x1862022294279808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768146.610584:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768146.610586:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099aa680 x1862022294279808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768171 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768146.610590:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768146.610592:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768146.610593:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768146.610594:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768146.610595:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768146.610596:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768146.610598:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768146.610600:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768146.610601:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768146.610603:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099aa680 x1862022294279808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768171 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768146.610607:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099aa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294279808:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768146.610609:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768146.610611:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ab800 x1862022294279936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768146.610615:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768146.610616:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768146.610617:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768146.610618:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768146.610619:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768146.610621:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294279936:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768146.610622:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768146.610623:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768146.610624:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768146.610625:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768146.610627:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f000. 02000000:00000001:2.0:1775768146.610629:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768146.610630:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768146.610632:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768146.610633:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768146.610635:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294279936, portal 10 00000100:00000001:2.0:1775768146.610636:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768146.610637:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640866816 : -112966068684800 : ffff9942099ab800) 00000100:00000040:2.0:1775768146.610640:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ab800 x1862022294279936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768171 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768146.610644:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768146.610646:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768146.610647:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768146.610648:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294279936, offset 0 00000400:00000200:2.0:1775768146.610650:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768146.610652:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768146.610655:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768146.610656:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768146.610658:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768146.610659:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768146.610661:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768146.610663:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225360:225360:256:4294967295] 192.168.202.156@tcp LPNI seq info [225356:225356:8:4294967295] 00000400:00000200:2.0:1775768146.610665:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768146.610669:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768146.610671:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768146.610673:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768146.610675:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768146.610677:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775768146.610679:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768146.610680:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000100:2.0:1775768146.610682:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768146.610683:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768146.610685:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768146.610687:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768146.610689:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768146.610690:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768146.610691:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768146.610693:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ab800 x1862022294279936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768171 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768146.610701:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768146.610702:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ab800 x1862022294279936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768171 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768146.610706:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768146.610707:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768146.610708:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768146.610709:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:2.0:1775768146.610711:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768146.610712:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300071435 00000100:00000001:2.0:1775768146.610713:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300072765 00000100:00000001:2.0:1775768146.610714:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300071435 00000100:00000001:2.0:1775768146.610716:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300072765 00000100:00000001:2.0:1775768146.610717:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768146.610718:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768146.610719:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768146.610721:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768146/real 1775768146] req@ffff9942099ab800 x1862022294279936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768146.610724:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768146.610725:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768146.610726:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768146.610727:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768146.610727:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768146.610728:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768146.610730:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ab800 x1862022294279936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768146.610734:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ab800 x1862022294279936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768146.610738:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768146.610739:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768146.610740:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768146.610741:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768146.610743:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ab800 x1862022294279936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768146.610747:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768146.610748:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ab800 x1862022294279936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768171 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768146.610752:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768146.610753:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768146.610754:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768146.610755:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768146.610755:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768146.610757:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768146.610758:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768146.610760:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768146.610761:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768146.610763:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ab800 x1862022294279936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768171 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768146.610767:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294279936:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768146.610769:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768146.610770:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768146.610771:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099aa680 x1862022294279808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768171 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768146.610775:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768146.610776:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768146.610777:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1e400. 02000000:00000001:2.0:1775768146.610779:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768146.610779:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768146.610781:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:2.0:1775768146.610782:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768146.610783:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b0c00. 02000000:00000001:2.0:1775768146.610785:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768146.610785:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768146.610786:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099aa680. 00000100:00000001:2.0:1775768146.610788:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768146.610789:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768146.610790:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768146.610791:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ab800 x1862022294279936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768171 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768146.610795:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768146.610795:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768146.610796:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f000. 02000000:00000001:2.0:1775768146.610797:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768146.610798:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768146.610798:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:2.0:1775768146.610799:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768146.610800:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b1000. 02000000:00000001:2.0:1775768146.610801:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768146.610802:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768146.610802:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ab800. 00000100:00000001:2.0:1775768146.610803:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768146.610804:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768146.610805:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768146.610807:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768146.610808:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768146.610809:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768146.610810:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768146.610810:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768146.610812:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099aa300 x1862022294280064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768146.610815:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768146.610816:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768146.610817:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768146.610818:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768146.610819:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768146.610821:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099aa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294280064:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768146.610823:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768146.610824:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768146.610825:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768146.610826:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768146.610827:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f000. 02000000:00000001:2.0:1775768146.610828:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768146.610830:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768146.610831:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768146.610833:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768146.610834:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294280064, portal 4 00000100:00000001:2.0:1775768146.610835:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768146.610836:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640861440 : -112966068690176 : ffff9942099aa300) 00000100:00000040:2.0:1775768146.610839:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099aa300 x1862022294280064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768171 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768146.610842:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768146.610844:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768146.610845:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768146.610846:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294280064, offset 0 00000400:00000200:2.0:1775768146.610848:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768146.610850:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768146.610851:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768146.610853:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768146.610854:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768146.610855:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768146.610857:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768146.610858:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225361:225361:256:4294967295] 192.168.202.156@tcp LPNI seq info [225357:225357:8:4294967295] 00000400:00000200:2.0:1775768146.610861:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768146.610864:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768146.610867:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768146.610868:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768146.610870:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768146.610872:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775768146.610874:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768146.610876:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000100:2.0:1775768146.610878:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768146.610879:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768146.610880:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768146.610882:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768146.610884:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768146.610885:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768146.610886:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768146.610887:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099aa300 x1862022294280064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768171 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768146.610892:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768146.610893:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099aa300 x1862022294280064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768171 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768146.610897:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768146.610898:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768146.610899:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768146.610901:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:2.0:1775768146.610902:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768146.610903:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300071458 00000100:00000001:2.0:1775768146.610905:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300072765 00000100:00000001:2.0:1775768146.610906:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300071458 00000100:00000001:2.0:1775768146.610907:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300072765 00000100:00000001:2.0:1775768146.610908:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768146.610909:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768146.610910:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768146.610912:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768146/real 1775768146] req@ffff9942099aa300 x1862022294280064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768146.610919:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768146.610920:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768146.610921:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768146.610922:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768146.610923:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768146.610924:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768146.610926:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099aa300 x1862022294280064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768146.610930:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099aa300 x1862022294280064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768146.610933:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768146.610933:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768146.610934:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768146.610935:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768146.610937:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099aa300 x1862022294280064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768146.610940:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768146.610942:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099aa300 x1862022294280064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768171 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768146.610945:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768146.610946:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768146.610947:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768146.610948:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768146.610949:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768146.610952:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768146.610953:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768146.610955:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768146.610956:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768146.610958:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099aa300 x1862022294280064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768171 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768146.610963:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099aa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294280064:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768146.610965:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768146.610967:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a8a80 x1862022294280192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768146.610970:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768146.610971:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768146.610972:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768146.610973:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768146.610973:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768146.610975:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294280192:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768146.610976:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768146.610977:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768146.610978:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768146.610979:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768146.610980:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1e400. 02000000:00000001:2.0:1775768146.610981:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768146.610982:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768146.610984:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768146.610985:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768146.610987:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294280192, portal 4 00000100:00000001:2.0:1775768146.610988:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768146.610989:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640855168 : -112966068696448 : ffff9942099a8a80) 00000100:00000040:2.0:1775768146.610992:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a8a80 x1862022294280192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768171 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768146.610995:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768146.610996:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768146.610997:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768146.610998:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294280192, offset 0 00000400:00000200:2.0:1775768146.611001:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768146.611002:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768146.611004:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768146.611005:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768146.611007:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768146.611008:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768146.611009:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768146.611011:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225362:225362:256:4294967295] 192.168.202.156@tcp LPNI seq info [225358:225358:8:4294967295] 00000400:00000200:2.0:1775768146.611013:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768146.611017:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768146.611019:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768146.611020:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768146.611022:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768146.611024:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775768146.611026:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768146.611028:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000100:2.0:1775768146.611030:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768146.611031:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768146.611032:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768146.611034:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768146.611036:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768146.611037:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768146.611038:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768146.611039:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a8a80 x1862022294280192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768171 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768146.611044:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768146.611045:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a8a80 x1862022294280192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768171 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768146.611048:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768146.611049:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768146.611050:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768146.611052:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:2.0:1775768146.611053:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768146.611055:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300071461 00000100:00000001:2.0:1775768146.611056:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300072765 00000100:00000001:2.0:1775768146.611058:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300071461 00000100:00000001:2.0:1775768146.611059:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300072765 00000100:00000001:2.0:1775768146.611060:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768146.611061:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768146.611062:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768146.611063:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768146/real 1775768146] req@ffff9942099a8a80 x1862022294280192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768146.611067:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768146.611068:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768146.611068:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768146.611070:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768146.611071:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768146.611072:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768146.611073:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a8a80 x1862022294280192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768146.611078:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a8a80 x1862022294280192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768146.611081:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768146.611082:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768146.611083:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768146.611083:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768146.611085:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a8a80 x1862022294280192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768146.611088:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768146.611090:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a8a80 x1862022294280192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768171 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768146.611093:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768146.611095:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768146.611096:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768146.611096:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768146.611097:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768146.611099:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768146.611101:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768146.611102:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768146.611103:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768146.611105:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a8a80 x1862022294280192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768171 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768146.611109:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294280192:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768146.611111:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768146.611112:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768146.611113:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099aa300 x1862022294280064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768171 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768146.611116:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768146.611117:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768146.611118:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f000. 02000000:00000001:2.0:1775768146.611119:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768146.611119:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768146.611120:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:2.0:1775768146.611121:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768146.611122:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b2000. 02000000:00000001:2.0:1775768146.611124:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768146.611125:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768146.611125:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099aa300. 00000100:00000001:2.0:1775768146.611127:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768146.611127:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768146.611128:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768146.611129:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a8a80 x1862022294280192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768171 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768146.611138:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768146.611139:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768146.611140:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1e400. 02000000:00000001:2.0:1775768146.611143:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768146.611144:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768146.611145:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:2.0:1775768146.611147:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768146.611148:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b3400. 02000000:00000001:2.0:1775768146.611150:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768146.611151:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768146.611152:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a8a80. 00000100:00000001:2.0:1775768146.611154:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768146.611170:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768146.611172:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768146.611175:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768146.611176:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768146.611177:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768146.611179:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768146.673255:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768146.673260:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768146.673264:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768146.673267:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768146.673268:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768146.673271:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768146.673273:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768146.673275:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768146.673276:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768146.673277:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768146.673279:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768146.673280:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768146.673281:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768146.673282:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768146.673283:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768146.673284:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768146.673285:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768146.673286:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768146.673288:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768146.673289:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768146.673289:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768146.673290:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768146.673291:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768146.673292:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768146.673292:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768146.673293:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768146.673295:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768146.673296:0:14953:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768146.673297:0:14953:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768146.673297:0:14953:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768146.673298:0:14953:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768146.673299:0:14953:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768146.673300:0:14953:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768146.673301:0:14953:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768146.673347:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768146.673350:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768146.673352:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768146.737230:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768146.737235:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768146.737237:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768146.737241:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768146.737243:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768146.737246:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768146.737248:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768146.737250:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768146.737251:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768147.633232:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768147.633238:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768147.633239:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768147.633245:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768147.633246:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768147.633248:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768147.633248:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768147.633249:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768147.633252:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768147.633253:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768147.633256:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768147.633256:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:3.0:1775768147.633256:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768147.633260:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768147.633260:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768147.633261:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768147.633267:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768147.633267:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768147.633272:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768147.633272:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768147.633275:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768147.633275:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768147.633277:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768147.633280:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768147.633281:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768147.633284:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768147.633287:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768147.697276:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768147.697283:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768147.697288:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768147.697290:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768147.697294:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768147.697295:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768147.697299:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768147.697300:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768147.697303:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768147.697304:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768147.697361:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768147.697364:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768147.697368:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768147.761252:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768147.761257:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768147.761258:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768147.761262:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768147.761264:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768147.761267:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768147.761269:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768147.761271:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768147.761273:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768147.953261:0:14953:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4300072765, at 4300072768 00000100:00000040:1.0:1775768147.953261:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4300072765, at 4300072768 00000100:00000010:1.0:1775768147.953275:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d6e40. 00000100:00000010:0.0:1775768147.953277:0:14953:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d6a80. 00000100:00000001:1.0:1775768148.657211:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768148.657214:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768148.657216:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768148.657219:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768148.657219:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768148.657221:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768148.657223:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775768148.657225:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768148.657226:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768148.657228:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768148.657230:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768148.657231:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768148.657234:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768148.657237:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768148.657237:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768148.657240:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768148.657242:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768148.657242:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768148.657245:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768148.657246:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768148.657251:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768148.657252:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768148.657257:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768148.657259:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768148.657262:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768148.657264:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768148.657266:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768148.721264:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768148.721270:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768148.721275:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768148.721276:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768148.721280:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768148.721281:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768148.721289:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768148.721291:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768148.721295:0:14953:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768148.721296:0:14953:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768148.721345:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768148.721348:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768148.721351:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768148.785243:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768148.785249:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768148.785251:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768148.785256:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768148.785259:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768148.785262:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768148.785265:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768148.785267:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768148.785270:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768149.681220:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768149.681222:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768149.681228:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768149.681228:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768149.681231:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768149.681232:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768149.681232:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768149.681236:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768149.681239:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768149.681240:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775768149.681240:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:1.0:1775768149.681243:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775768149.681246:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768149.681247:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768149.681247:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768149.681252:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768149.681253:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768149.681253:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768149.681258:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768149.681260:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768149.681263:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768149.681265:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768149.681268:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768149.681272:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768149.681278:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768149.681281:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768149.681284:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768149.745278:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768149.745284:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768149.745289:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768149.745291:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768149.745294:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768149.745295:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768149.745298:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768149.745299:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768149.745301:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768149.745302:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768149.745348:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768149.745352:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768149.745355:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768149.809265:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768149.809271:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768149.809273:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768149.809278:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768149.809282:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768149.809286:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768149.809288:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768149.809290:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768149.809293:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768150.705210:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768150.705210:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768150.705215:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768150.705216:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768150.705216:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768150.705219:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768150.705222:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775768150.705223:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768150.705229:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768150.705229:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768150.705233:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768150.705233:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768150.705238:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768150.705241:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768150.705244:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768150.705245:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768150.705248:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768150.705250:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768150.705299:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768150.705301:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768150.705302:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768150.705304:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768150.705313:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768150.705315:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768150.705317:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768150.705319:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768150.705323:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768150.769237:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768150.769241:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768150.769247:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768150.769248:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768150.769251:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768150.769252:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768150.769255:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768150.769257:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768150.769259:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768150.769260:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768150.769313:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768150.769317:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768150.769320:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768150.833215:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768150.833220:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768150.833222:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768150.833225:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768150.833228:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768150.833231:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768150.833233:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768150.833234:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768150.833236:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768151.729218:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768151.729225:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775768151.729227:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775768151.729230:0:36:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775768151.729234:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:0.0:1775768151.729238:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768151.729238:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:3.0:1775768151.729240:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5404 00000100:00000001:0.0:1775768151.729243:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768151.729244:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768151.729245:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768151.729245:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768151.729246:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768151.729249:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:0.0:1775768151.729251:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000400:00000200:3.0:1775768151.729251:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00100000:1.0:1775768151.729252:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775768151.729252:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768151.729256:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768151.729257:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:2.0:1775768151.729257:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000400:00000200:3.0:1775768151.729259:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:0.0:1775768151.729260:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768151.729261:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768151.729261:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1775768151.729263:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1775768151.729265:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768151.729266:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000040:3.0:1775768151.729268:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768151.729269:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768151.729273:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1775768151.729274:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768151.729274:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768151.729278:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768151.729281:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768151.729281:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000040:3.0:1775768151.729282:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768151.729283:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768151.729285:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:2.0:1775768151.729286:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768151.729291:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775768151.729293:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:3.0:1775768151.729296:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775768151.729298:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000080:00000001:0.0:1775768151.729299:0:13471:0:(rw.c:1481:ll_writepages()) Process entered 00000020:00000001:3.0:1775768151.729300:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000080:00000001:0.0:1775768151.729301:0:13471:0:(file.c:5250:cl_sync_file_range()) Process entered 00000020:00000001:3.0:1775768151.729302:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:0.0:1775768151.729304:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00000040:3.0:1775768151.729305:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768151.729307:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775768151.729308:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768151.729309:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775768151.729310:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:3.0:1775768151.729312:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775768151.729314:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768151.729317:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775768151.729319:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775768151.729321:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768151.729322:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000020:00000010:0.0:1775768151.729324:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff9942103bed80. 00000100:00000001:3.0:1775768151.729324:0:36:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775768151.729328:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8e00. 00000020:00000010:0.0:1775768151.729331:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0ec00. 00000020:00000040:3.0:1775768151.729331:0:36:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:3.0:1775768151.729335:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775768151.729337:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1775768151.729338:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210b0f600. 00000100:00000001:3.0:1775768151.729339:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775768151.729342:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000020:00000010:0.0:1775768151.729344:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942078a3d80. 02000000:00000001:3.0:1775768151.729346:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:0.0:1775768151.729352:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff9942086e6468. 02000000:00000010:3.0:1775768151.729352:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0400. 00020000:00000010:0.0:1775768151.729360:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994209592f00. 00000100:00000001:3.0:1775768151.729360:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:0.0:1775768151.729365:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff99422035b880. 00000100:00100000:3.0:1775768151.729367:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8e00 x1862022294280320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000080:00000010:0.0:1775768151.729370:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994207e3e150. 00000080:00000010:0.0:1775768151.729374:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b2238. 00000020:00000010:0.0:1775768151.729381:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0fc00. 00000100:00000001:3.0:1775768151.729385:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000008:00000010:0.0:1775768151.729387:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff99420906be40. 00000100:00000001:3.0:1775768151.729389:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:0.0:1775768151.729393:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff9942103ccfc0. 00000100:00000040:3.0:1775768151.729394:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000864a6866] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8e00 x1862022294280320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00008000:00000010:0.0:1775768151.729400:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076afa0. 00000080:00000010:0.0:1775768151.729405:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942107690f0. 00000100:00000001:3.0:1775768151.729408:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768151.729409:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107752095112 : -112965957456504 : ffff9942103bed88) 00000100:00080000:3.0:1775768151.729410:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00001000:0.0:1775768151.729414:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff9942103bed80 00000100:00080000:3.0:1775768151.729415:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775768151.729417:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775768151.729418:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000020:00000001:0.0:1775768151.729419:0:13471:0:(cl_object.c:151:cl_object_top()) ffff9942078a76c0 -> ffff9942078a76c0 00000100:00000001:3.0:1775768151.729420:0:36:0:(import.c:484:import_select_connection()) Process entered 00000020:00000001:0.0:1775768151.729422:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000100:00080000:3.0:1775768151.729424:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5404 00000400:00000200:3.0:1775768151.729439:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000080:00000001:0.0:1775768151.729440:0:13471:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:0.0:1775768151.729442:0:13471:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000400:00000200:3.0:1775768151.729442:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775768151.729444:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000020:00000001:0.0:1775768151.729447:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000100:00000040:3.0:1775768151.729448:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000001:0.0:1775768151.729449:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00000001:3.0:1775768151.729450:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000080:00000001:0.0:1775768151.729451:0:13471:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768151.729454:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00020000:00000002:0.0:1775768151.729455:0:13471:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017a:0x0]io ffff9942077b2298 type 6 ignore/verify layout 1/0 00000100:00000040:3.0:1775768151.729457:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00020000:00000002:0.0:1775768151.729459:0:13471:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207c1f9b0) owner 0000000000000000/ffff994220080000 00000100:00000001:2.0:1775768151.729459:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768151.729462:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1775768151.729464:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00020000:00000001:0.0:1775768151.729465:0:13471:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1775768151.729467:0:13471:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00000020:00000001:3.0:1775768151.729467:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775768151.729470:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00020000:00000001:0.0:1775768151.729471:0:13471:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00000100:00000040:2.0:1775768151.729471:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a8e00 x1862022294280320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:0.0:1775768151.729475:0:13471:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768151.729475:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00020000:00000001:0.0:1775768151.729479:0:13471:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000040:3.0:1775768151.729479:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00020000:00000001:0.0:1775768151.729485:0:13471:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00000020:00000001:3.0:1775768151.729485:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00020000:00000001:0.0:1775768151.729487:0:13471:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768151.729491:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:0.0:1775768151.729492:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000100:00000001:2.0:1775768151.729492:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000001:0.0:1775768151.729494:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1775768151.729496:0:13471:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00000100:00000001:2.0:1775768151.729496:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1775768151.729500:0:13471:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207c1f9b0) owner 0000000000000000/ffff994220080000 00000100:00000040:3.0:1775768151.729500:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 02000000:00000001:2.0:1775768151.729502:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:0.0:1775768151.729503:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768151.729505:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:0.0:1775768151.729507:0:13471:0:(cl_io.c:823:cl_io_loop()) Process entered 02000000:00000001:2.0:1775768151.729507:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1775768151.729508:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000100:00000040:3.0:1775768151.729510:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00020000:00000001:0.0:1775768151.729511:0:13471:0:(lov_io.c:836:lov_io_iter_init()) Process entered 02000000:00000001:2.0:1775768151.729512:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00020000:00200000:0.0:1775768151.729515:0:13471:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00000001:3.0:1775768151.729516:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00020000:00000002:0.0:1775768151.729518:0:13471:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00000100:00100000:2.0:1775768151.729520:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294280320:192.168.202.156@tcp:250:kworker.0 00000020:00000040:3.0:1775768151.729521:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00020000:00000001:0.0:1775768151.729522:0:13471:0:(lov_io.c:155:lov_sub_get()) Process entered 00020000:00000001:0.0:1775768151.729524:0:13471:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000020:00000001:0.0:1775768151.729526:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00000001:2.0:1775768151.729528:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000010:0.0:1775768151.729529:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff9942103be000. 00000100:00080000:3.0:1775768151.729529:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000020:00000010:0.0:1775768151.729533:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0e000. 02000000:00000001:2.0:1775768151.729533:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1775768151.729534:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1775768151.729536:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210b0ea00. 02000000:00000001:2.0:1775768151.729537:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768151.729538:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000020:00000010:0.0:1775768151.729539:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942078a30c0. 02000000:00000001:2.0:1775768151.729543:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000010:0.0:1775768151.729544:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff9942086e4e18. 00020000:00000010:0.0:1775768151.729548:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994209592e40. 02000000:00000010:2.0:1775768151.729549:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1e400. 00008000:00000010:0.0:1775768151.729550:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994220358000. 02000000:00000001:3.0:1775768151.729550:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768151.729554:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768151.729555:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1775768151.729556:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994207e3e3f0. 00000100:00000001:3.0:1775768151.729559:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000080:00000010:0.0:1775768151.729561:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b0ff8. 00000008:00100000:3.0:1775768151.729563:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000400:00000200:2.0:1775768151.729565:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000010:0.0:1775768151.729567:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0fa00. 00000008:00000001:3.0:1775768151.729568:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768151.729570:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775768151.729571:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000008:00000010:0.0:1775768151.729574:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff99420906a6e0. 00000100:00000010:3.0:1775768151.729574:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ab480. 00000400:00000010:2.0:1775768151.729579:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000020:00000040:3.0:1775768151.729579:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00020000:00000010:0.0:1775768151.729580:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff9942103cc480. 02000000:00000001:3.0:1775768151.729584:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00008000:00000010:0.0:1775768151.729585:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076adb0. 02000000:00000001:3.0:1775768151.729586:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1775768151.729588:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294280320, portal 25 00000100:00000001:3.0:1775768151.729589:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000080:00000010:0.0:1775768151.729591:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff994210769870. 02000000:00000001:3.0:1775768151.729592:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775768151.729594:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768151.729596:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107752091656 : -112965957459960 : ffff9942103be008) 00000100:00000001:2.0:1775768151.729598:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 02000000:00000010:3.0:1775768151.729599:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0000. 00000020:00001000:0.0:1775768151.729600:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff9942103be000 00000100:00000001:2.0:1775768151.729601:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640856064 : -112966068695552 : ffff9942099a8e00) 00000100:00000001:3.0:1775768151.729604:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768151.729605:0:13471:0:(cl_object.c:151:cl_object_top()) ffff9942078a7788 -> ffff9942078a76c0 00000020:00000001:0.0:1775768151.729609:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000100:00000040:2.0:1775768151.729610:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a8e00 x1862022294280320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768176 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768151.729611:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ab480 x1862022294280448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000001:0.0:1775768151.729613:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1775768151.729616:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:0.0:1775768151.729619:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768151.729625:0:13471:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768151.729629:0:13471:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107752153128 : -112965957398488 : ffff9942103cd028) 00000020:00000001:0.0:1775768151.729634:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000020:00000001:0.0:1775768151.729638:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768151.729640:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775768151.729641:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00020000:00000001:0.0:1775768151.729642:0:13471:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768151.729643:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768151.729646:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768151.729648:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000100:00000040:3.0:1775768151.729649:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000067a0ba74] to pc [ptlrpcd_rcv+-1] req@ffff9942099ab480 x1862022294280448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775768151.729651:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00000400:00000010:2.0:1775768151.729654:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00020000:00000001:0.0:1775768151.729655:0:13471:0:(lov_io.c:1108:lov_io_lock()) Process entered 00020000:00000001:0.0:1775768151.729659:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000200:2.0:1775768151.729661:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294280320, offset 0 00000020:00000001:0.0:1775768151.729662:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:0.0:1775768151.729665:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000100:00000001:3.0:1775768151.729665:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768151.729668:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768151.729670:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000001:0.0:1775768151.729673:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768151.729673:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00020000:00000001:0.0:1775768151.729678:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768151.729678:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775768151.729681:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00020000:00000001:0.0:1775768151.729682:0:13471:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768151.729683:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00080000:3.0:1775768151.729684:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000020:00000001:0.0:1775768151.729688:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000400:00000200:2.0:1775768151.729689:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000020:00000001:0.0:1775768151.729691:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768151.729691:0:36:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:2.0:1775768151.729694:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:0.0:1775768151.729695:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768151.729697:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5404 00000020:00000001:0.0:1775768151.729700:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00000400:00000200:2.0:1775768151.729700:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00020000:00000001:0.0:1775768151.729705:0:13471:0:(lov_io.c:1114:lov_io_start()) Process entered 00000400:00000200:2.0:1775768151.729706:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00020000:00000001:0.0:1775768151.729708:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000400:00000200:2.0:1775768151.729709:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000020:00000001:0.0:1775768151.729711:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00000400:00000200:2.0:1775768151.729714:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225363:225363:256:4294967295] 192.168.202.156@tcp LPNI seq info [225359:225359:8:4294967295] 00000008:00000001:0.0:1775768151.729716:0:13471:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000400:00000200:2.0:1775768151.729719:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00000001:0.0:1775768151.729720:0:13471:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000400:00000200:3.0:1775768151.729724:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00000001:0.0:1775768151.729727:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000400:00000200:3.0:1775768151.729730:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000008:00000020:0.0:1775768151.729731:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000400:00000200:2.0:1775768151.729733:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000008:00000001:0.0:1775768151.729735:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775768151.729735:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775768151.729740:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000008:00000001:0.0:1775768151.729741:0:13471:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000100:00000001:3.0:1775768151.729742:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000008:00000001:0.0:1775768151.729744:0:13471:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000100:00000040:3.0:1775768151.729745:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000800:00000200:2.0:1775768151.729747:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000008:00000001:0.0:1775768151.729748:0:13471:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107884691392 : -112965824860224 : ffff994218232fc0) 00000100:00000001:3.0:1775768151.729749:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775768151.729752:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775768151.729754:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000008:00000020:0.0:1775768151.729755:0:13471:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994218232fc0 ready 0|-|- wr 1|-|+ rd 0|- 10 in flight 00000020:00000001:3.0:1775768151.729756:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000800:00000010:2.0:1775768151.729757:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000020:00000040:3.0:1775768151.729758:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000008:00000001:0.0:1775768151.729763:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000020:00000001:3.0:1775768151.729763:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:3.0:1775768151.729765:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000008:00000020:0.0:1775768151.729766:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:0.0:1775768151.729769:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1775768151.729771:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000040:3.0:1775768151.729771:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000008:00000001:0.0:1775768151.729773:0:13471:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000100:00000001:3.0:1775768151.729774:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775768151.729778:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000008:00000020:0.0:1775768151.729779:0:13471:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994210675880 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000800:00000200:2.0:1775768151.729779:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000100:00000001:3.0:1775768151.729779:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775768151.729782:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:3.0:1775768151.729787:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000008:00000001:0.0:1775768151.729790:0:13471:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768151.729791:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775768151.729792:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000008:00000001:0.0:1775768151.729795:0:13471:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768151.729796:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:2.1:1775768151.729797:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 02000000:00000001:3.0:1775768151.729799:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775768151.729802:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1775768151.729805:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000100:00000001:3.0:1775768151.729805:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000020:00000001:0.0:1775768151.729808:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1775768151.729809:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000008:00100000:3.0:1775768151.729809:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775768151.729814:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768151.729815:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00020000:00000001:0.0:1775768151.729816:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768151.729817:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768151.729821:0:13471:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775768151.729823:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a9180. 00000020:00000001:0.0:1775768151.729825:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768151.729826:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000020:00000001:0.0:1775768151.729829:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00020000:00000001:0.0:1775768151.729832:0:13471:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00000400:00000200:2.0:1775768151.729833:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000020:00000040:3.0:1775768151.729834:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00020000:00000001:0.0:1775768151.729836:0:13471:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000400:00000010:2.0:1775768151.729836:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000020:00000001:0.0:1775768151.729839:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00000100:00000001:2.0:1775768151.729841:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:0.0:1775768151.729842:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 02000000:00000001:3.0:1775768151.729843:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00020000:00000001:0.0:1775768151.729844:0:13471:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1775768151.729846:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a8e00 x1862022294280320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768176 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1775768151.729846:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768151.729851:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775768151.729853:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00020000:00000001:0.0:1775768151.729856:0:13471:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 02000000:00000001:3.0:1775768151.729857:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768151.729859:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00000020:00000001:0.0:1775768151.729863:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 02000000:00000010:3.0:1775768151.729864:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2c00. 00000100:00000001:2.0:1775768151.729865:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00020000:00000001:0.0:1775768151.729866:0:13471:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00020000:00000001:0.0:1775768151.729869:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000001:3.0:1775768151.729870:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768151.729872:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000100:00000040:2.0:1775768151.729872:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a8e00 x1862022294280320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768176 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775768151.729875:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:0.0:1775768151.729877:0:13471:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768151.729877:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a9180 x1862022294280576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00020000:00000001:0.0:1775768151.729883:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768151.729888:0:13471:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000020:00000001:0.0:1775768151.729891:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:0.0:1775768151.729894:0:13471:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000020:00000001:0.0:1775768151.729898:0:13471:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000100:00000001:2.0:1775768151.729899:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768151.729902:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:0.0:1775768151.729905:0:13471:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00000100:00000001:2.0:1775768151.729905:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00020000:00000001:0.0:1775768151.729909:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000001:3.0:1775768151.729909:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000020:00000001:0.0:1775768151.729912:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000100:00000001:2.0:1775768151.729912:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768151.729912:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768151.729916:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000100:00000040:3.0:1775768151.729917:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bdc2586] to pc [ptlrpcd_rcv+-1] req@ffff9942099a9180 x1862022294280576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:0.0:1775768151.729919:0:13471:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768151.729920:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768151.729925:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4300077885 00020000:00000001:0.0:1775768151.729926:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768151.729930:0:13471:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000100:00000010:2.0:1775768151.729931:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d6f00. 00000020:00000001:0.0:1775768151.729933:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000100:00000001:3.0:1775768151.729934:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768151.729938:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00080000:3.0:1775768151.729938:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768151.729942:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1775768151.729944:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4300077885 00000020:00000001:0.0:1775768151.729945:0:13471:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775768151.729948:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d6720. 00000020:00000001:0.0:1775768151.729949:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00000100:00000001:2.0:1775768151.729951:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1775768151.729951:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00020000:00000001:0.0:1775768151.729953:0:13471:0:(lov_io.c:675:lov_io_fini()) Process entered 00000100:00080000:3.0:1775768151.729955:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00020000:00000001:0.0:1775768151.729956:0:13471:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000100:00000001:2.0:1775768151.729956:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1775768151.729959:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00000100:00000001:3.0:1775768151.729959:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775768151.729960:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768151.729963:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000100:00000001:2.0:1775768151.729963:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00080000:3.0:1775768151.729964:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5404 00000020:00001000:0.0:1775768151.729967:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff9942103be000 00000100:00100000:2.0:1775768151.729971:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768151/real 1775768151] req@ffff9942099a8e00 x1862022294280320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1775768151.729971:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000010:0.0:1775768151.729974:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210b0ea00. 00000400:00000200:3.0:1775768151.729976:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775768151.729980:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000020:00000010:0.0:1775768151.729981:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942078a30c0. 00000100:00000040:3.0:1775768151.729985:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000008:00000010:0.0:1775768151.729988:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff9942086e4e18. 00000100:00000001:3.0:1775768151.729990:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000001:2.0:1775768151.729995:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00020000:00000010:0.0:1775768151.729996:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994209592e40. 00000100:00000040:3.0:1775768151.729998:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00008000:00000010:0.0:1775768151.730001:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994220358000. 00000080:00000010:0.0:1775768151.730005:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994207e3e3f0. 00000100:00000001:3.0:1775768151.730005:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000001:2.0:1775768151.730007:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:3.0:1775768151.730010:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000080:00000010:0.0:1775768151.730011:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b0ff8. 00000020:00000040:3.0:1775768151.730013:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000400:00000010:2.0:1775768151.730014:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000020:00000010:0.0:1775768151.730016:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0e000. 00000020:00000001:3.0:1775768151.730017:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:2.0:1775768151.730019:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000020:00000040:3.0:1775768151.730021:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000008:00000010:0.0:1775768151.730022:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff99420906a6e0. 00000400:00000010:2.0:1775768151.730022:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000020:00000001:3.0:1775768151.730025:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:2.0:1775768151.730028:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00020000:00000010:0.0:1775768151.730029:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff9942103cc480. 00000020:00000001:3.0:1775768151.730029:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00008000:00000010:0.0:1775768151.730036:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076adb0. 00000100:00000200:2.0:1775768151.730036:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a8e00 x1862022294280320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775768151.730037:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000080:00000010:0.0:1775768151.730042:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff994210769870. 00000100:00000001:3.0:1775768151.730044:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000010:0.0:1775768151.730048:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0fa00. 00000100:00000040:3.0:1775768151.730049:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000010:0.0:1775768151.730054:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff9942103be000. 00000100:00000001:3.0:1775768151.730055:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000200:2.0:1775768151.730059:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a8e00 x1862022294280320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:0.0:1775768151.730060:0:13471:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00000020:00000040:3.0:1775768151.730064:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00020000:00000001:0.0:1775768151.730065:0:13471:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000100:00080000:3.0:1775768151.730070:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775768151.730072:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775768151.730072:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768151.730074:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1775768151.730075:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1775768151.730076:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:3.0:1775768151.730077:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000001:0.0:1775768151.730078:0:13471:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000100:00000001:2.0:1775768151.730079:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1775768151.730082:0:13471:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 02000000:00000001:3.0:1775768151.730082:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768151.730086:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a8e00 x1862022294280320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768151.730087:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000080:00000001:0.0:1775768151.730089:0:13471:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000008:00100000:3.0:1775768151.730090:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000020:00000001:0.0:1775768151.730091:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:3.0:1775768151.730095:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1775768151.730096:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff9942103bed80 00000020:00000010:0.0:1775768151.730101:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210b0f600. 00000100:00000001:2.0:1775768151.730101:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768151.730101:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775768151.730106:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8700. 00000020:00000010:0.0:1775768151.730107:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942078a3d80. 00000100:00000040:2.0:1775768151.730109:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a8e00 x1862022294280320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768176 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1775768151.730110:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00000008:00000010:0.0:1775768151.730111:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff9942086e6468. 00020000:00000010:0.0:1775768151.730117:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994209592f00. 02000000:00000001:3.0:1775768151.730117:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775768151.730120:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768151.730121:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00008000:00000010:0.0:1775768151.730122:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff99422035b880. 00000100:00000001:3.0:1775768151.730125:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000080:00000010:0.0:1775768151.730126:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994207e3e150. 00000100:00000001:2.0:1775768151.730128:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1775768151.730128:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000080:00000010:0.0:1775768151.730129:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b2238. 02000000:00000001:3.0:1775768151.730131:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768151.730133:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000010:0.0:1775768151.730134:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0ec00. 00000100:00000001:2.0:1775768151.730135:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:3.0:1775768151.730137:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2800. 00000008:00000010:0.0:1775768151.730138:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff99420906be40. 00000100:00000001:2.0:1775768151.730140:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00020000:00000010:0.0:1775768151.730144:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff9942103ccfc0. 00000100:00000001:2.0:1775768151.730144:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775768151.730144:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:0.0:1775768151.730149:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076afa0. 00000100:00080000:2.0:1775768151.730150:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000080:00000010:0.0:1775768151.730153:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942107690f0. 00000100:00100000:3.0:1775768151.730179:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8700 x1862022294280704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000010:0.0:1775768151.730181:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0fc00. 00000020:00000010:0.0:1775768151.730187:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff9942103bed80. 00000100:00000001:3.0:1775768151.730189:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00080000:2.0:1775768151.730191:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000080:00000001:0.0:1775768151.730193:0:13471:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768151.730193:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768151.730196:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000080:00000001:0.0:1775768151.730199:0:13471:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768151.730199:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000faa25e3b] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8700 x1862022294280704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775768151.730205:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a8e00 x1862022294280320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768176 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000001:0.0:1775768151.730209:0:13471:0:(rw.c:1481:ll_writepages()) Process entered 00000100:00000001:3.0:1775768151.730213:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1775768151.730214:0:13471:0:(file.c:5250:cl_sync_file_range()) Process entered 00000020:00000001:0.0:1775768151.730217:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00080000:3.0:1775768151.730219:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000010:0.0:1775768151.730222:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff9942103bed80. 00000100:00100000:2.0:1775768151.730227:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294280320:192.168.202.156@tcp:250:kworker.0 00000020:00000010:0.0:1775768151.730228:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0fc00. 00000100:00080000:3.0:1775768151.730228:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775768151.730231:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000020:00000010:0.0:1775768151.730233:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210b0ec00. 00000100:00080000:3.0:1775768151.730235:0:36:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000020:00000010:0.0:1775768151.730237:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942078a3d80. 00000100:00000001:2.0:1775768151.730238:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768151.730239:0:36:0:(import.c:484:import_select_connection()) Process entered 00000008:00000010:0.0:1775768151.730240:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff9942086e6468. 00020000:00000010:0.0:1775768151.730243:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994209592f00. 00000100:00000001:2.0:1775768151.730243:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00080000:3.0:1775768151.730245:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5404 00008000:00000010:0.0:1775768151.730247:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff99422035b880. 00000100:00000040:2.0:1775768151.730250:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a8e00 x1862022294280320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768176 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000010:0.0:1775768151.730251:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994207e3e150. 00000400:00000200:3.0:1775768151.730255:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000080:00000010:0.0:1775768151.730256:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b2238. 00000020:00000010:0.0:1775768151.730261:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0f600. 00000400:00000200:3.0:1775768151.730263:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000008:00000010:0.0:1775768151.730266:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff99420906be40. 00000100:00000001:2.0:1775768151.730267:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 00000100:00080000:3.0:1775768151.730267:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00020000:00000010:0.0:1775768151.730271:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff9942103ccfc0. 02000000:00000001:2.0:1775768151.730271:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000040:3.0:1775768151.730273:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00008000:00000010:0.0:1775768151.730275:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076afa0. 02000000:00000010:2.0:1775768151.730276:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1e400. 00000100:00000001:3.0:1775768151.730276:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000080:00000010:0.0:1775768151.730280:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942107690f0. 00000100:00000040:3.0:1775768151.730280:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000001:2.0:1775768151.730282:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768151.730283:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107752095112 : -112965957456504 : ffff9942103bed88) 00000100:00000001:3.0:1775768151.730284:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775768151.730286:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00001000:0.0:1775768151.730289:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff9942103bed80 00000020:00000001:3.0:1775768151.730289:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775768151.730290:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000040:3.0:1775768151.730292:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:0.0:1775768151.730293:0:13471:0:(cl_object.c:151:cl_object_top()) ffff9942078a7f58 -> ffff9942078a7f58 00000020:00000001:2.0:1775768151.730295:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 00000020:00000001:3.0:1775768151.730295:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000001:0.0:1775768151.730296:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000040:3.0:1775768151.730297:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000080:00000001:0.0:1775768151.730299:0:13471:0:(vvp_io.c:1812:vvp_io_init()) Process entered 02000000:00000010:2.0:1775768151.730300:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b0400. 00000020:00000001:3.0:1775768151.730300:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000080:00200000:0.0:1775768151.730301:0:13471:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017b:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:3.0:1775768151.730304:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:0.0:1775768151.730307:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 02000000:00000001:2.0:1775768151.730307:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000040:3.0:1775768151.730308:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000001:0.0:1775768151.730310:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 02000000:00000001:2.0:1775768151.730311:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1775768151.730311:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000080:00000001:0.0:1775768151.730313:0:13471:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775768151.730314:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a8e00. 00000100:00000040:3.0:1775768151.730316:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00020000:00000002:0.0:1775768151.730318:0:13471:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017b:0x0]io ffff9942077b2298 type 6 ignore/verify layout 1/0 00000100:00000001:2.0:1775768151.730320:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775768151.730320:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:2.0:1775768151.730323:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00020000:00000002:0.0:1775768151.730324:0:13471:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207c1f000) owner 0000000000000000/ffff994220080000 00000020:00000040:3.0:1775768151.730325:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00000001:2.0:1775768151.730328:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00020000:00000001:0.0:1775768151.730330:0:13471:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00000100:00080000:3.0:1775768151.730331:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00020000:00000001:0.0:1775768151.730333:0:13471:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00000100:00000001:2.0:1775768151.730335:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00020000:00000001:0.0:1775768151.730336:0:13471:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00000100:00000001:3.0:1775768151.730337:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768151.730338:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768151.730339:0:13471:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768151.730341:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1775768151.730343:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00020000:00000001:0.0:1775768151.730344:0:13471:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768151.730346:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1775768151.730346:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1775768151.730348:0:13471:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00000100:00000001:2.0:1775768151.730350:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00020000:00000001:0.0:1775768151.730351:0:13471:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768151.730351:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768151.730356:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ab480 x1862022294280448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775768151.730357:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000100:00000001:3.0:1775768151.730357:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000020:00000001:0.0:1775768151.730360:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000008:00100000:3.0:1775768151.730361:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00020000:00000001:0.0:1775768151.730362:0:13471:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00020000:00000002:0.0:1775768151.730365:0:13471:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207c1f000) owner 0000000000000000/ffff994220080000 00000008:00000001:3.0:1775768151.730368:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768151.730371:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768151.730372:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768151.730377:0:13471:0:(cl_io.c:823:cl_io_loop()) Process entered 00000100:00000001:2.0:1775768151.730378:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000001:0.0:1775768151.730380:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000100:00000010:3.0:1775768151.730380:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8380. 00000100:00000001:2.0:1775768151.730381:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768151.730384:0:13471:0:(lov_io.c:836:lov_io_iter_init()) Process entered 02000000:00000001:2.0:1775768151.730385:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:3.0:1775768151.730385:0:36:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00020000:00200000:0.0:1775768151.730387:0:13471:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 02000000:00000001:2.0:1775768151.730389:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775768151.730391:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00020000:00000002:0.0:1775768151.730392:0:13471:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 02000000:00000001:2.0:1775768151.730394:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768151.730394:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768151.730397:0:13471:0:(lov_io.c:155:lov_sub_get()) Process entered 00000100:00000001:3.0:1775768151.730397:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00020000:00000001:0.0:1775768151.730400:0:13471:0:(lov_io.c:104:lov_io_sub_init()) Process entered 02000000:00000001:3.0:1775768151.730400:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:2.0:1775768151.730401:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ab480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294280448:192.168.202.156@tcp:38:kworker.0 00000020:00000001:0.0:1775768151.730403:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 02000000:00000001:3.0:1775768151.730403:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768151.730409:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000010:0.0:1775768151.730410:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff9942103be000. 02000000:00000010:3.0:1775768151.730410:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0400. 02000000:00000001:2.0:1775768151.730411:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768151.730414:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768151.730415:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1775768151.730416:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0fa00. 02000000:00000001:2.0:1775768151.730420:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000010:0.0:1775768151.730422:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210b0e000. 00000100:00100000:3.0:1775768151.730423:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8380 x1862022294280832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000010:2.0:1775768151.730436:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1e400. 00000020:00000010:0.0:1775768151.730437:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942078a30c0. 02000000:00000001:2.0:1775768151.730442:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:0.0:1775768151.730443:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff9942086e4e18. 00020000:00000010:0.0:1775768151.730448:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994209592e40. 00000100:00000001:3.0:1775768151.730449:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00008000:00000010:0.0:1775768151.730451:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994220358000. 00000400:00000200:2.0:1775768151.730453:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775768151.730453:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1775768151.730456:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994207e3e3f0. 00000400:00000010:2.0:1775768151.730459:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000100:00000040:3.0:1775768151.730460:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055f5621a] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8380 x1862022294280832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000010:0.0:1775768151.730461:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b0ff8. 00000020:00000010:0.0:1775768151.730465:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0ea00. 00000400:00000010:2.0:1775768151.730467:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000008:00000010:0.0:1775768151.730471:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff99420906a6e0. 00000100:00000200:2.0:1775768151.730474:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294280448, portal 10 00020000:00000010:0.0:1775768151.730476:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff9942103cc480. 00000100:00000001:3.0:1775768151.730478:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:0.0:1775768151.730479:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076adb0. 00000100:00000001:2.0:1775768151.730479:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768151.730482:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640865920 : -112966068685696 : ffff9942099ab480) 00000100:00000040:3.0:1775768151.730482:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5414) 00000080:00000010:0.0:1775768151.730483:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff994210769870. 00000020:00000001:0.0:1775768151.730487:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107752091656 : -112965957459960 : ffff9942103be008) 00000020:00001000:0.0:1775768151.730489:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff9942103be000 00000020:00000001:0.0:1775768151.730491:0:13471:0:(cl_object.c:151:cl_object_top()) ffff9942078a75f8 -> ffff9942078a7f58 00000020:00000001:0.0:1775768151.730493:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000100:00000040:2.0:1775768151.730494:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ab480 x1862022294280448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768176 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775768151.730495:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1775768151.730496:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:0.0:1775768151.730498:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768151.730501:0:13471:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768151.730504:0:13471:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107752153128 : -112965957398488 : ffff9942103cd028) 00000020:00000001:0.0:1775768151.730507:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000020:00000001:0.0:1775768151.730508:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768151.730511:0:13471:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768151.730512:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:0.0:1775768151.730513:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768151.730515:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00020000:00000001:0.0:1775768151.730517:0:13471:0:(lov_io.c:1108:lov_io_lock()) Process entered 00000100:00000040:2.0:1775768151.730518:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00020000:00000001:0.0:1775768151.730519:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:0.0:1775768151.730520:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:0.0:1775768151.730521:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000400:00000010:2.0:1775768151.730521:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000020:00000001:0.0:1775768151.730522:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768151.730525:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1775768151.730525:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294280448, offset 0 00020000:00000001:0.0:1775768151.730526:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768151.730530:0:13471:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768151.730532:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000400:00000200:2.0:1775768151.730533:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000020:00000001:0.0:1775768151.730534:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768151.730536:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768151.730539:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00000400:00000200:2.0:1775768151.730539:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:0.0:1775768151.730541:0:13471:0:(lov_io.c:1114:lov_io_start()) Process entered 00020000:00000001:0.0:1775768151.730542:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000400:00000200:2.0:1775768151.730544:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000020:00000001:0.0:1775768151.730545:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00000008:00000001:0.0:1775768151.730547:0:13471:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000008:00000001:0.0:1775768151.730549:0:13471:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000400:00000200:2.0:1775768151.730549:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768151.730552:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000008:00000001:0.0:1775768151.730553:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:0.0:1775768151.730556:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000400:00000200:2.0:1775768151.730556:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000008:00000001:0.0:1775768151.730559:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1775768151.730559:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000008:00000001:0.0:1775768151.730564:0:13471:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000400:00000200:2.0:1775768151.730564:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225364:225364:256:4294967295] 192.168.202.156@tcp LPNI seq info [225360:225360:8:4294967295] 00000008:00000001:0.0:1775768151.730567:0:13471:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000400:00000200:2.0:1775768151.730569:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00000001:0.0:1775768151.730571:0:13471:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107884688256 : -112965824863360 : ffff994218232380) 00000008:00000020:0.0:1775768151.730576:0:13471:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994218232380 ready 0|-|- wr 1|-|+ rd 0|- 9 in flight 00000400:00000200:2.0:1775768151.730576:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000008:00000001:0.0:1775768151.730583:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000800:00000200:2.0:1775768151.730584:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000008:00000020:0.0:1775768151.730586:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000800:00000010:2.0:1775768151.730587:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000008:00000001:0.0:1775768151.730588:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1775768151.730592:0:13471:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000800:00000200:2.0:1775768151.730592:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768151.730595:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000008:00000020:0.0:1775768151.730596:0:13471:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994210674fc0 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000800:00000200:2.1:1775768151.730602:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000008:00000001:0.0:1775768151.730603:0:13471:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.1:1775768151.730605:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000008:00000001:0.0:1775768151.730607:0:13471:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1775768151.730611:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000020:00000001:0.0:1775768151.730612:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1775768151.730612:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00020000:00000001:0.0:1775768151.730616:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768151.730617:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00020000:00000001:0.0:1775768151.730621:0:13471:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768151.730622:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000020:00000001:0.0:1775768151.730624:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768151.730625:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000020:00000001:0.0:1775768151.730626:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00000400:00000010:2.0:1775768151.730627:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00020000:00000001:0.0:1775768151.730628:0:13471:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00020000:00000001:0.0:1775768151.730631:0:13471:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000100:00000001:2.0:1775768151.730631:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:0.0:1775768151.730634:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000001:0.0:1775768151.730635:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00020000:00000001:0.0:1775768151.730636:0:13471:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1775768151.730636:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ab480 x1862022294280448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768176 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:0.0:1775768151.730640:0:13471:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000020:00000001:0.0:1775768151.730642:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00000020:00000001:0.0:1775768151.730644:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00020000:00000001:0.0:1775768151.730646:0:13471:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00020000:00000001:0.0:1775768151.730648:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000001:2.0:1775768151.730648:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000020:00000001:0.0:1775768151.730649:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000020:00000001:0.0:1775768151.730651:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:0.0:1775768151.730652:0:13471:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768151.730652:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ab480 x1862022294280448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768176 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:0.0:1775768151.730656:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768151.730659:0:13471:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000020:00000001:0.0:1775768151.730661:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:0.0:1775768151.730663:0:13471:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000020:00000001:0.0:1775768151.730665:0:13471:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000001:0.0:1775768151.730666:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:0.0:1775768151.730668:0:13471:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00000100:00000001:2.0:1775768151.730669:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768151.730670:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:0.0:1775768151.730671:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000100:00000001:2.0:1775768151.730673:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:0.0:1775768151.730674:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00020000:00000001:0.0:1775768151.730676:0:13471:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768151.730678:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768151.730679:0:13471:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000100:00000001:2.0:1775768151.730679:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768151.730681:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000020:00000001:0.0:1775768151.730683:0:13471:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768151.730684:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1775768151.730685:0:13471:0:(lov_io.c:675:lov_io_fini()) Process entered 00000100:00000040:2.0:1775768151.730685:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00020000:00000001:0.0:1775768151.730687:0:13471:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:0.0:1775768151.730688:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:0.0:1775768151.730690:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1775768151.730692:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff9942103be000 00000400:00000200:2.0:1775768151.730693:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000010:0.0:1775768151.730694:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210b0e000. 00000100:00000001:2.0:1775768151.730697:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300076563 00000020:00000010:0.0:1775768151.730698:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942078a30c0. 00000008:00000010:0.0:1775768151.730702:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff9942086e4e18. 00000100:00000001:2.0:1775768151.730703:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300077885 00020000:00000010:0.0:1775768151.730704:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994209592e40. 00000100:00000001:2.0:1775768151.730707:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300076563 00008000:00000010:0.0:1775768151.730708:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994220358000. 00000080:00000010:0.0:1775768151.730711:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994207e3e3f0. 00000100:00000001:2.0:1775768151.730712:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300077885 00000080:00000010:0.0:1775768151.730714:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b0ff8. 00000100:00000001:2.0:1775768151.730715:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1775768151.730718:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0fa00. 00000100:00000001:2.0:1775768151.730718:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:0.0:1775768151.730722:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff99420906a6e0. 00000100:00000001:2.0:1775768151.730722:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00020000:00000010:0.0:1775768151.730725:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff9942103cc480. 00000100:00100000:2.0:1775768151.730726:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768151/real 1775768151] req@ffff9942099ab480 x1862022294280448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00008000:00000010:0.0:1775768151.730727:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076adb0. 00000080:00000010:0.0:1775768151.730732:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff994210769870. 00000020:00000010:0.0:1775768151.730735:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0ea00. 00000020:00000010:0.0:1775768151.730739:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff9942103be000. 00000400:00000001:2.0:1775768151.730740:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00020000:00000001:0.0:1775768151.730741:0:13471:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00000400:00000001:2.0:1775768151.730743:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00020000:00000001:0.0:1775768151.730744:0:13471:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000400:00000010:2.0:1775768151.730745:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000080:00000001:0.0:1775768151.730746:0:13471:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000080:00200000:0.0:1775768151.730749:0:13471:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000400:00000200:2.0:1775768151.730749:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768151.730752:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000080:00000001:0.0:1775768151.730754:0:13471:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000100:00000001:2.0:1775768151.730756:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:0.0:1775768151.730757:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1775768151.730761:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff9942103bed80 00000100:00000200:2.0:1775768151.730763:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ab480 x1862022294280448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1775768151.730764:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210b0ec00. 00000020:00000010:0.0:1775768151.730769:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942078a3d80. 00000008:00000010:0.0:1775768151.730774:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff9942086e6468. 00020000:00000010:0.0:1775768151.730779:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994209592f00. 00000100:00000200:2.0:1775768151.730781:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ab480 x1862022294280448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00008000:00000010:0.0:1775768151.730783:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff99422035b880. 00000080:00000010:0.0:1775768151.730787:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994207e3e150. 00000080:00000010:0.0:1775768151.730793:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b2238. 00000100:00000001:2.0:1775768151.730793:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768151.730795:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768151.730797:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000010:0.0:1775768151.730798:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0fc00. 00000100:00000001:2.0:1775768151.730798:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00000010:0.0:1775768151.730803:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff99420906be40. 00000100:00100000:2.0:1775768151.730804:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ab480 x1862022294280448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000010:0.0:1775768151.730809:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff9942103ccfc0. 00000100:00000001:2.0:1775768151.730811:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00008000:00000010:0.0:1775768151.730814:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076afa0. 00000100:00000040:2.0:1775768151.730815:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ab480 x1862022294280448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768176 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000010:0.0:1775768151.730820:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942107690f0. 00000020:00000010:0.0:1775768151.730824:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0f600. 00000020:00000010:0.0:1775768151.730828:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff9942103bed80. 00000100:00000001:2.0:1775768151.730829:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000080:00000001:0.0:1775768151.730832:0:13471:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768151.730834:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1775768151.730836:0:13471:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768151.730837:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768151.730840:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1775768151.730842:0:13471:0:(rw.c:1481:ll_writepages()) Process entered 00000100:00000001:2.0:1775768151.730843:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000080:00000001:0.0:1775768151.730845:0:13471:0:(file.c:5250:cl_sync_file_range()) Process entered 00000100:00000001:2.0:1775768151.730847:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:0.0:1775768151.730848:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:0.0:1775768151.730852:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff9942103bed80. 00000100:00080000:2.0:1775768151.730852:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000020:00000010:0.0:1775768151.730857:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0f600. 00000100:00080000:2.0:1775768151.730859:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000020:00000010:0.0:1775768151.730862:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210b0fc00. 00000100:00000001:2.0:1775768151.730863:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000010:0.0:1775768151.730866:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942078a3d80. 00000100:00000040:2.0:1775768151.730869:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ab480 x1862022294280448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768176 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000010:0.0:1775768151.730872:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff9942086e6468. 00020000:00000010:0.0:1775768151.730876:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994209592f00. 00008000:00000010:0.0:1775768151.730881:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff99422035b880. 00000100:00100000:2.0:1775768151.730884:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ab480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294280448:192.168.202.156@tcp:38:kworker.0 00000080:00000010:0.0:1775768151.730886:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994207e3e150. 00000080:00000010:0.0:1775768151.730891:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b2238. 00000100:00000001:2.0:1775768151.730892:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000010:0.0:1775768151.730897:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0ec00. 00000100:00000040:2.0:1775768151.730898:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a9180 x1862022294280576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000010:0.0:1775768151.730901:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff99420906be40. 00020000:00000010:0.0:1775768151.730906:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff9942103ccfc0. 00008000:00000010:0.0:1775768151.730911:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076afa0. 00000080:00000010:0.0:1775768151.730915:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942107690f0. 00000020:00000001:0.0:1775768151.730920:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107752095112 : -112965957456504 : ffff9942103bed88) 00000100:00000001:2.0:1775768151.730921:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768151.730922:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1775768151.730925:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff9942103bed80 02000000:00000001:2.0:1775768151.730925:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768151.730928:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1775768151.730929:0:13471:0:(cl_object.c:151:cl_object_top()) ffff99420791a850 -> ffff99420791a850 02000000:00000001:2.0:1775768151.730930:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768151.730934:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000100:00100000:2.0:1775768151.730934:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294280576:192.168.202.156@tcp:38:kworker.0 00000080:00000001:0.0:1775768151.730937:0:13471:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000100:00000001:2.0:1775768151.730938:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000080:00200000:0.0:1775768151.730940:0:13471:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017f:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 02000000:00000001:2.0:1775768151.730940:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768151.730942:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768151.730945:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:0.0:1775768151.730947:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 02000000:00000010:2.0:1775768151.730948:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f000. 00000020:00000001:0.0:1775768151.730949:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 02000000:00000001:2.0:1775768151.730951:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1775768151.730952:0:13471:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768151.730956:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000002:0.0:1775768151.730958:0:13471:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017f:0x0]io ffff9942077b2298 type 6 ignore/verify layout 1/0 00000400:00000010:2.0:1775768151.730959:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768151.730962:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00020000:00000002:0.0:1775768151.730964:0:13471:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207dfbd90) owner 0000000000000000/ffff994220080000 00000100:00000200:2.0:1775768151.730965:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294280576, portal 10 00000100:00000001:2.0:1775768151.730967:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768151.730969:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640856960 : -112966068694656 : ffff9942099a9180) 00020000:00000001:0.0:1775768151.730970:0:13471:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1775768151.730973:0:13471:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1775768151.730976:0:13471:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00000100:00000040:2.0:1775768151.730976:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a9180 x1862022294280576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768176 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:0.0:1775768151.730979:0:13471:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768151.730983:0:13471:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768151.730986:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00020000:00000001:0.0:1775768151.730988:0:13471:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1775768151.730991:0:13471:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768151.730991:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000020:00000001:0.0:1775768151.730995:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000400:00000010:2.0:1775768151.730996:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000020:00000001:0.0:1775768151.730997:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1775768151.731000:0:13471:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00000100:00000200:2.0:1775768151.731001:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294280576, offset 0 00020000:00000002:0.0:1775768151.731002:0:13471:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207dfbd90) owner 0000000000000000/ffff994220080000 00000020:00000001:0.0:1775768151.731007:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768151.731007:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000020:00000001:0.0:1775768151.731011:0:13471:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:0.0:1775768151.731013:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00020000:00000001:0.0:1775768151.731015:0:13471:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00000400:00000200:2.0:1775768151.731015:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00200000:0.0:1775768151.731018:0:13471:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000400:00000200:2.0:1775768151.731022:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00020000:00000002:0.0:1775768151.731023:0:13471:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:0.0:1775768151.731028:0:13471:0:(lov_io.c:155:lov_sub_get()) Process entered 00000400:00000200:2.0:1775768151.731028:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00020000:00000001:0.0:1775768151.731031:0:13471:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000400:00000200:2.0:1775768151.731033:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000020:00000001:0.0:1775768151.731034:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000400:00000200:2.0:1775768151.731038:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000020:00000010:0.0:1775768151.731039:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff9942103be000. 00000400:00000200:2.0:1775768151.731040:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000020:00000010:0.0:1775768151.731045:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0ea00. 00000400:00000200:2.0:1775768151.731045:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225365:225365:256:4294967295] 192.168.202.156@tcp LPNI seq info [225361:225361:8:4294967295] 00000020:00000010:0.0:1775768151.731050:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210b0fa00. 00000400:00000200:2.0:1775768151.731051:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000010:0.0:1775768151.731056:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942078a30c0. 00000400:00000200:2.0:1775768151.731059:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000008:00000010:0.0:1775768151.731061:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff9942086e4e18. 00000800:00000200:2.0:1775768151.731064:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00020000:00000010:0.0:1775768151.731067:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994209592e40. 00000800:00000010:2.0:1775768151.731068:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00008000:00000010:0.0:1775768151.731071:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994220358000. 00000800:00000200:2.0:1775768151.731072:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768151.731076:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000080:00000010:0.0:1775768151.731078:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994207e3e3f0. 00000800:00000200:2.1:1775768151.731081:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000080:00000010:0.0:1775768151.731082:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b0ff8. 00000800:00000200:2.1:1775768151.731084:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000020:00000010:0.0:1775768151.731086:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210b0e000. 00000008:00000010:0.0:1775768151.731090:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff99420906a6e0. 00000800:00000100:2.0:1775768151.731090:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768151.731093:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00020000:00000010:0.0:1775768151.731094:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff9942103cc480. 00000400:00000200:2.0:1775768151.731096:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00008000:00000010:0.0:1775768151.731097:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076adb0. 00000080:00000010:0.0:1775768151.731101:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff994210769870. 00000400:00000200:2.0:1775768151.731103:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000020:00000001:0.0:1775768151.731104:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107752091656 : -112965957459960 : ffff9942103be008) 00000400:00000200:2.0:1775768151.731106:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000020:00001000:0.0:1775768151.731107:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff9942103be000 00000400:00000010:2.0:1775768151.731108:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000020:00000001:0.0:1775768151.731109:0:13471:0:(cl_object.c:151:cl_object_top()) ffff99420791a210 -> ffff99420791a850 00000100:00000001:2.0:1775768151.731111:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:0.0:1775768151.731112:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:0.0:1775768151.731115:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000100:00000200:2.0:1775768151.731115:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a9180 x1862022294280576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768176 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775768151.731117:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:0.0:1775768151.731119:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768151.731122:0:13471:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768151.731125:0:13471:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107752153128 : -112965957398488 : ffff9942103cd028) 00000020:00000001:0.0:1775768151.731128:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000100:00000001:2.0:1775768151.731128:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000020:00000001:0.0:1775768151.731129:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768151.731132:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a9180 x1862022294280576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768176 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:0.0:1775768151.731133:0:13471:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768151.731136:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768151.731138:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00020000:00000001:0.0:1775768151.731140:0:13471:0:(lov_io.c:1108:lov_io_lock()) Process entered 00020000:00000001:0.0:1775768151.731142:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:0.0:1775768151.731144:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:0.0:1775768151.731145:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000100:00000001:2.0:1775768151.731145:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768151.731148:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768151.731148:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:0.0:1775768151.731150:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768151.731151:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768151.731152:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768151.731177:0:13471:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768151.731178:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000020:00000001:0.0:1775768151.731179:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:0.0:1775768151.731182:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768151.731185:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768151.731186:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:0.0:1775768151.731190:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00000100:00000001:2.0:1775768151.731191:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300076556 00020000:00000001:0.0:1775768151.731192:0:13471:0:(lov_io.c:1114:lov_io_start()) Process entered 00020000:00000001:0.0:1775768151.731195:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000001:2.0:1775768151.731196:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300077885 00000020:00000001:0.0:1775768151.731197:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00000100:00000001:2.0:1775768151.731199:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300076556 00000008:00000001:0.0:1775768151.731200:0:13471:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000008:00000001:0.0:1775768151.731202:0:13471:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000100:00000001:2.0:1775768151.731204:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300077885 00000008:00000001:0.0:1775768151.731206:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000100:00000001:2.0:1775768151.731207:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00000020:0.0:1775768151.731208:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:0.0:1775768151.731211:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768151.731212:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1775768151.731214:0:13471:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000100:00000001:2.0:1775768151.731215:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000008:00000001:0.0:1775768151.731216:0:13471:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000008:00000001:0.0:1775768151.731219:0:13471:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107754938304 : -112965954613312 : ffff994210674fc0) 00000100:00100000:2.0:1775768151.731222:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768151/real 1775768151] req@ffff9942099a9180 x1862022294280576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000020:0.0:1775768151.731224:0:13471:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994210674fc0 ready 0|-|- wr 1|-|+ rd 0|- 9 in flight 00000008:00000001:0.0:1775768151.731231:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:0.0:1775768151.731234:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:0.0:1775768151.731236:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:2.0:1775768151.731239:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000008:00000001:0.0:1775768151.731240:0:13471:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000400:00000001:2.0:1775768151.731242:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000008:00000020:0.0:1775768151.731243:0:13471:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994218232380 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000400:00000010:2.0:1775768151.731245:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000008:00000001:0.0:1775768151.731250:0:13471:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1775768151.731251:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000008:00000001:0.0:1775768151.731253:0:13471:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775768151.731254:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000020:00000001:0.0:1775768151.731257:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768151.731258:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00020000:00000001:0.0:1775768151.731261:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768151.731265:0:13471:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1775768151.731265:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a9180 x1862022294280576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775768151.731269:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768151.731273:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00020000:00000001:0.0:1775768151.731275:0:13471:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00020000:00000001:0.0:1775768151.731277:0:13471:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000020:00000001:0.0:1775768151.731279:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000001:0.0:1775768151.731281:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00000100:00000200:2.0:1775768151.731283:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a9180 x1862022294280576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:0.0:1775768151.731284:0:13471:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768151.731288:0:13471:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000020:00000001:0.0:1775768151.731290:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00000020:00000001:0.0:1775768151.731292:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00020000:00000001:0.0:1775768151.731294:0:13471:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00020000:00000001:0.0:1775768151.731297:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000001:2.0:1775768151.731298:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:0.0:1775768151.731299:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000100:00000001:2.0:1775768151.731301:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1775768151.731302:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:0.0:1775768151.731305:0:13471:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768151.731305:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768151.731307:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00020000:00000001:0.0:1775768151.731308:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768151.731312:0:13471:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000020:00000001:0.0:1775768151.731314:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000100:00100000:2.0:1775768151.731314:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a9180 x1862022294280576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775768151.731315:0:13471:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000020:00000001:0.0:1775768151.731318:0:13471:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000001:0.0:1775768151.731320:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:0.0:1775768151.731323:0:13471:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00020000:00000001:0.0:1775768151.731325:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:0.0:1775768151.731328:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000100:00000001:2.0:1775768151.731329:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1775768151.731330:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00020000:00000001:0.0:1775768151.731333:0:13471:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768151.731334:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a9180 x1862022294280576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768176 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:0.0:1775768151.731336:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1775768151.731340:0:13471:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000020:00000001:0.0:1775768151.731342:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000020:00000001:0.0:1775768151.731345:0:13471:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768151.731349:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00000100:00000001:2.0:1775768151.731350:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00020000:00000001:0.0:1775768151.731351:0:13471:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:0.0:1775768151.731355:0:13471:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000100:00000001:2.0:1775768151.731355:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1775768151.731357:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:0.0:1775768151.731359:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000100:00000001:2.0:1775768151.731360:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00001000:0.0:1775768151.731361:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff9942103be000 00000100:00000001:2.0:1775768151.731362:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:0.0:1775768151.731364:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210b0fa00. 00000100:00000001:2.0:1775768151.731365:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768151.731370:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000010:0.0:1775768151.731371:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942078a30c0. 00000008:00000010:0.0:1775768151.731376:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff9942086e4e18. 00000100:00080000:2.0:1775768151.731376:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00020000:00000010:0.0:1775768151.731382:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994209592e40. 00000100:00080000:2.0:1775768151.731383:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00008000:00000010:0.0:1775768151.731387:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994220358000. 00000100:00000001:2.0:1775768151.731387:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000080:00000010:0.0:1775768151.731392:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994207e3e3f0. 00000100:00000040:2.0:1775768151.731395:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a9180 x1862022294280576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768176 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000010:0.0:1775768151.731396:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b0ff8. 00000020:00000010:0.0:1775768151.731401:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0ea00. 00000008:00000010:0.0:1775768151.731406:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff99420906a6e0. 00020000:00000010:0.0:1775768151.731411:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff9942103cc480. 00000100:00100000:2.0:1775768151.731412:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294280576:192.168.202.156@tcp:38:kworker.0 00008000:00000010:0.0:1775768151.731414:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076adb0. 00000080:00000010:0.0:1775768151.731417:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff994210769870. 00000020:00000010:0.0:1775768151.731421:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0e000. 00000100:00000001:2.0:1775768151.731421:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000010:0.0:1775768151.731422:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff9942103be000. 00020000:00000001:0.0:1775768151.731436:0:13471:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:0.0:1775768151.731439:0:13471:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:0.0:1775768151.731441:0:13471:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000080:00200000:0.0:1775768151.731443:0:13471:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017f:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00000040:2.0:1775768151.731443:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a8700 x1862022294280704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000001:0.0:1775768151.731449:0:13471:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1775768151.731451:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1775768151.731453:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff9942103bed80 00000100:00000001:2.0:1775768151.731454:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000010:0.0:1775768151.731456:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210b0fc00. 00000100:00000001:2.0:1775768151.731457:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1775768151.731459:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942078a3d80. 02000000:00000001:2.0:1775768151.731461:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000010:0.0:1775768151.731462:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff9942086e6468. 00020000:00000010:0.0:1775768151.731465:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994209592f00. 02000000:00000001:2.0:1775768151.731465:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00008000:00000010:0.0:1775768151.731467:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff99422035b880. 00000080:00000010:0.0:1775768151.731471:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994207e3e150. 02000000:00000001:2.0:1775768151.731471:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1775768151.731474:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b2238. 00000020:00000010:0.0:1775768151.731478:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0f600. 00000100:00100000:2.0:1775768151.731478:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294280704:192.168.202.156@tcp:8:kworker.0 00000008:00000010:0.0:1775768151.731482:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff99420906be40. 00020000:00000010:0.0:1775768151.731486:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff9942103ccfc0. 00000100:00000001:2.0:1775768151.731486:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00008000:00000010:0.0:1775768151.731489:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076afa0. 02000000:00000001:2.0:1775768151.731489:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768151.731492:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1775768151.731493:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942107690f0. 00000020:00000010:0.0:1775768151.731496:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210b0ec00. 02000000:00000001:2.0:1775768151.731497:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000010:0.0:1775768151.731500:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff9942103bed80. 00000080:00000001:0.0:1775768151.731502:0:13471:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:2.0:1775768151.731502:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f800. 00000080:00000001:0.0:1775768151.731504:0:13471:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768151.731507:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768151.731512:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768151.731526:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768151.731530:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768151.731532:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294280704, portal 4 00000100:00000001:2.0:1775768151.731534:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768151.731536:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640854272 : -112966068697344 : ffff9942099a8700) 00000100:00000040:2.0:1775768151.731541:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a8700 x1862022294280704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768176 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768151.731547:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768151.731549:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768151.731550:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768151.731552:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294280704, offset 0 00000400:00000200:2.0:1775768151.731554:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768151.731557:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768151.731559:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768151.731561:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768151.731563:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768151.731564:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768151.731566:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768151.731568:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225366:225366:256:4294967295] 192.168.202.156@tcp LPNI seq info [225362:225362:8:4294967295] 00000400:00000200:2.0:1775768151.731570:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768151.731573:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768151.731576:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768151.731578:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768151.731580:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768151.731582:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775768151.731584:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768151.731586:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000100:2.0:1775768151.731588:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768151.731589:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768151.731591:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768151.731593:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768151.731594:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768151.731595:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768151.731597:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768151.731598:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a8700 x1862022294280704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768176 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768151.731603:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768151.731605:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a8700 x1862022294280704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768176 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768151.731608:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768151.731610:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768151.731611:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768151.731613:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:2.0:1775768151.731615:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768151.731616:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300076579 00000100:00000001:2.0:1775768151.731618:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300077885 00000100:00000001:2.0:1775768151.731619:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300076579 00000100:00000001:2.0:1775768151.731620:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300077885 00000100:00000001:2.0:1775768151.731621:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768151.731622:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768151.731624:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768151.731625:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768151/real 1775768151] req@ffff9942099a8700 x1862022294280704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768151.731630:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768151.731631:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768151.731632:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768151.731633:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768151.731634:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768151.731635:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768151.731636:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a8700 x1862022294280704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768151.731640:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a8700 x1862022294280704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768151.731644:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768151.731645:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768151.731646:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768151.731647:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768151.731648:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a8700 x1862022294280704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768151.731652:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768151.731655:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a8700 x1862022294280704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768176 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768151.731661:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768151.731663:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768151.731665:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768151.731666:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768151.731667:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768151.731670:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768151.731672:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768151.731675:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768151.731678:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768151.731681:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a8700 x1862022294280704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768176 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768151.731689:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294280704:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768151.731693:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768151.731694:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768151.731697:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ab480 x1862022294280448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768176 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768151.731703:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768151.731704:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768151.731705:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1e400. 02000000:00000001:2.0:1775768151.731707:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768151.731708:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768151.731709:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:2.0:1775768151.731710:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768151.731711:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b0000. 02000000:00000001:2.0:1775768151.731713:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768151.731714:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768151.731715:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ab480. 00000100:00000001:2.0:1775768151.731717:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768151.731718:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768151.731719:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768151.731721:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a9180 x1862022294280576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768176 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768151.731725:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768151.731725:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768151.731726:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f000. 02000000:00000001:2.0:1775768151.731727:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768151.731728:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768151.731728:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:2.0:1775768151.731729:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768151.731730:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b2c00. 02000000:00000001:2.0:1775768151.731731:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768151.731731:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768151.731732:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a9180. 00000100:00000001:2.0:1775768151.731733:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768151.731734:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768151.731734:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768151.731736:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a8700 x1862022294280704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768176 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768151.731739:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768151.731740:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768151.731740:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f800. 02000000:00000001:2.0:1775768151.731741:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768151.731742:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768151.731742:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:2.0:1775768151.731743:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768151.731744:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b2800. 02000000:00000001:2.0:1775768151.731745:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768151.731745:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768151.731746:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a8700. 00000100:00000001:2.0:1775768151.731747:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768151.731748:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768151.731750:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768151.731753:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768151.731754:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768151.731756:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768151.731757:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768151.731758:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768151.731762:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a8380 x1862022294280832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768151.731768:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768151.731769:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768151.731771:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768151.731772:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768151.731774:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768151.731777:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294280832:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768151.731780:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768151.731782:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768151.731783:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768151.731784:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768151.731787:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f800. 02000000:00000001:2.0:1775768151.731789:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768151.731792:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768151.731793:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768151.731795:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768151.731797:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294280832, portal 4 00000100:00000001:2.0:1775768151.731798:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768151.731799:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640853376 : -112966068698240 : ffff9942099a8380) 00000100:00000040:2.0:1775768151.731803:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a8380 x1862022294280832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768176 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768151.731806:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768151.731808:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768151.731809:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768151.731811:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294280832, offset 0 00000400:00000200:2.0:1775768151.731814:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768151.731817:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768151.731820:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768151.731823:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768151.731825:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768151.731828:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768151.731830:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768151.731832:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225367:225367:256:4294967295] 192.168.202.156@tcp LPNI seq info [225363:225363:8:4294967295] 00000400:00000200:2.0:1775768151.731836:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768151.731842:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768151.731846:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768151.731849:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768151.731852:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768151.731854:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775768151.731858:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768151.731860:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000100:2.0:1775768151.731863:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768151.731864:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768151.731866:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768151.731869:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768151.731872:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768151.731873:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768151.731875:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768151.731878:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a8380 x1862022294280832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768176 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768151.731885:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768151.731887:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a8380 x1862022294280832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768176 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768151.731893:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768151.731895:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768151.731898:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768151.731900:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:2.0:1775768151.731903:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768151.731906:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300076582 00000100:00000001:2.0:1775768151.731908:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300077885 00000100:00000001:2.0:1775768151.731910:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300076582 00000100:00000001:2.0:1775768151.731913:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300077885 00000100:00000001:2.0:1775768151.731915:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768151.731917:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768151.731920:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768151.731923:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768151/real 1775768151] req@ffff9942099a8380 x1862022294280832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768151.731928:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768151.731930:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768151.731931:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768151.731933:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768151.731934:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768151.731936:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768151.731937:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a8380 x1862022294280832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768151.731943:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a8380 x1862022294280832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768151.731946:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768151.731947:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768151.731948:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768151.731949:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768151.731950:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a8380 x1862022294280832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768151.731954:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768151.731956:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a8380 x1862022294280832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768176 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768151.731960:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768151.731961:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768151.731962:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768151.731963:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768151.731964:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768151.731966:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768151.731968:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768151.731970:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768151.731971:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768151.731973:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a8380 x1862022294280832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768176 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768151.731978:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294280832:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768151.731979:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768151.731980:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768151.731982:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a8380 x1862022294280832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768176 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768151.731985:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768151.731986:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768151.731987:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f800. 02000000:00000001:2.0:1775768151.731989:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768151.731990:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768151.731991:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:2.0:1775768151.731993:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768151.731994:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b0400. 02000000:00000001:2.0:1775768151.731996:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768151.731997:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768151.731998:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a8380. 00000100:00000001:2.0:1775768151.732001:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768151.732003:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768151.732005:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768151.732008:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768151.732009:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768151.732011:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768151.732013:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768151.793266:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768151.793274:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768151.793276:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768151.793279:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768151.793284:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768151.793286:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768151.793288:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768151.793290:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768151.793293:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768151.793295:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768151.793298:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768151.793299:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768151.793302:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768151.793303:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768151.793306:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768151.793307:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768151.793355:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768151.793358:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768151.793360:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768151.857265:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768151.857272:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768151.857273:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768151.857284:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768151.857286:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768151.857290:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768151.857292:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768151.857293:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768151.857295:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768152.753210:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768152.753210:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768152.753216:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768152.753216:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768152.753220:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768152.753220:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768152.753225:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:1.0:1775768152.753226:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:2.0:1775768152.753226:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775768152.753231:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768152.753232:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768152.753232:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768152.753235:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768152.753236:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768152.753236:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768152.753242:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768152.753242:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768152.753244:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768152.753246:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768152.753248:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768152.753248:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768152.753249:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768152.753251:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768152.753255:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768152.753255:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768152.753259:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768152.753263:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768152.817226:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768152.817230:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768152.817234:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768152.817234:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768152.817237:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768152.817238:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768152.817240:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768152.817240:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768152.817242:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768152.817243:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768152.817293:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768152.817296:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768152.817299:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768152.881208:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768152.881212:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768152.881214:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768152.881218:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768152.881220:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768152.881223:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768152.881225:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768152.881227:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768152.881228:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768153.073213:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4300077885, at 4300077888 00000100:00000040:0.0:1775768153.073221:0:15036:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4300077885, at 4300077888 00000100:00000010:0.0:1775768153.073229:0:15036:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d6f00. 00000100:00000010:1.0:1775768153.073230:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d6720. 00000100:00000001:0.0:1775768153.777242:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768153.777242:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768153.777242:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768153.777250:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768153.777250:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768153.777251:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768153.777252:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768153.777254:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768153.777255:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768153.777259:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:1.0:1775768153.777261:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:2.0:1775768153.777261:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775768153.777264:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768153.777266:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768153.777267:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768153.777268:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768153.777270:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768153.777271:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768153.777272:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768153.777276:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768153.777279:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768153.777279:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768153.777281:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768153.777283:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768153.777284:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768153.777287:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768153.777289:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768153.841258:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768153.841263:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768153.841269:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768153.841271:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768153.841274:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768153.841275:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768153.841278:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768153.841280:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768153.841283:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768153.841284:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768153.841330:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768153.841333:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768153.841336:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768153.905254:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768153.905259:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768153.905261:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768153.905265:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768153.905267:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768153.905270:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768153.905272:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768153.905274:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768153.905277:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768154.801318:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768154.801318:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768154.801327:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768154.801327:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768154.801331:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768154.801331:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768154.801334:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768154.801339:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:1.0:1775768154.801340:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:2.0:1775768154.801340:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768154.801340:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768154.801346:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768154.801348:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768154.801349:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768154.801351:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768154.801353:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768154.801354:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768154.801355:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768154.801357:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768154.801361:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768154.801364:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768154.801364:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768154.801368:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768154.801371:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768154.801372:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768154.801375:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768154.801380:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768154.865269:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768154.865273:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768154.865277:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768154.865279:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768154.865285:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768154.865286:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768154.865288:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768154.865289:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768154.865291:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768154.865292:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768154.865371:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768154.865377:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768154.865382:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768154.929246:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768154.929250:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768154.929252:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768154.929256:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768154.929258:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768154.929261:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768154.929263:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768154.929264:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768154.929266:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768155.825245:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768155.825245:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768155.825253:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768155.825253:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768155.825255:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768155.825257:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:1.0:1775768155.825261:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775768155.825262:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768155.825264:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768155.825266:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768155.825267:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1775768155.825269:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768155.825271:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768155.825271:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768155.825272:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768155.825276:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768155.825276:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768155.825277:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768155.825279:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768155.825282:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768155.825282:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768155.825283:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768155.825285:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768155.825289:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768155.825289:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768155.825292:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768155.825295:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768155.889254:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768155.889262:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768155.889269:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768155.889277:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768155.889281:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768155.889282:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768155.889286:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768155.889287:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768155.889290:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768155.889291:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768155.889343:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768155.889346:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768155.889348:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768155.953259:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768155.953265:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768155.953266:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768155.953272:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768155.953276:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768155.953280:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768155.953282:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768155.953285:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768155.953287:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768156.849241:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768156.849241:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768156.849247:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768156.849247:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768156.849250:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768156.849251:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768156.849257:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:2.0:1775768156.849258:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768156.849262:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768156.849266:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768156.849267:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768156.849271:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768156.849271:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768156.849275:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768156.849283:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768156.849287:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768156.849287:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768156.849288:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768156.849291:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768156.849291:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768156.849295:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768156.849296:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768156.849299:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768156.849302:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768156.849304:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768156.849306:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768156.849307:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768156.849336:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768156.849345:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775768156.849349:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768156.849354:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768156.849359:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768156.849369:0:13471:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5409 00000400:00000200:0.0:1775768156.849393:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768156.849402:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768156.849406:0:13471:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768156.849411:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768156.849415:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768156.849419:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768156.849423:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768156.849433:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768156.849436:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:0.0:1775768156.849440:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768156.849443:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:0.0:1775768156.849447:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:0.0:1775768156.849451:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:0.0:1775768156.849455:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768156.849458:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768156.849461:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768156.849464:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768156.849468:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:0.0:1775768156.849474:0:13471:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768156.849477:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768156.849480:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768156.849484:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768156.849487:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768156.849489:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775768156.849492:0:13471:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768156.849501:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fe300. 00000020:00000040:0.0:1775768156.849505:0:13471:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:0.0:1775768156.849510:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768156.849512:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768156.849515:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768156.849519:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768156.849530:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768156.849540:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220029400. 00000100:00000001:0.0:1775768156.849551:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768156.849561:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fe300 x1862022294280960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768156.849572:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768156.849577:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768156.849583:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008a1e3639] to pc [ptlrpcd_rcv+-1] req@ffff9942099fe300 x1862022294280960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768156.849610:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768156.849613:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768156.849617:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775768156.849618:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768156.849621:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768156.849624:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768156.849628:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5409 00000400:00000200:0.0:1775768156.849632:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768156.849633:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768156.849639:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000400:00000200:0.0:1775768156.849640:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:1.0:1775768156.849642:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00080000:0.0:1775768156.849646:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768156.849653:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768156.849657:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768156.849657:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fe300 x1862022294280960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775768156.849667:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768156.849675:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768156.849685:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:1.0:1775768156.849686:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768156.849691:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768156.849695:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768156.849697:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:0.0:1775768156.849698:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 02000000:00000001:1.0:1775768156.849700:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768156.849701:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00100000:1.0:1775768156.849705:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294280960:192.168.202.156@tcp:250:kworker.0 00000020:00000040:0.0:1775768156.849706:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:0.0:1775768156.849715:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000001:1.0:1775768156.849715:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000001:0.0:1775768156.849721:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 02000000:00000001:1.0:1775768156.849723:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768156.849726:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768156.849731:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 02000000:00000001:1.0:1775768156.849731:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1775768156.849736:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000010:1.0:1775768156.849737:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543fc00. 00000100:00000040:0.0:1775768156.849741:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000001:1.0:1775768156.849742:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768156.849752:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000200:1.0:1775768156.849753:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768156.849757:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000020:00000040:0.0:1775768156.849765:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000400:00000010:1.0:1775768156.849768:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00080000:0.0:1775768156.849774:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768156.849778:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775768156.849778:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294280960, portal 25 00000100:00000001:1.0:1775768156.849784:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 02000000:00000001:0.0:1775768156.849785:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1775768156.849786:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641205504 : -112966068346112 : ffff9942099fe300) 02000000:00000001:0.0:1775768156.849793:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1775768156.849793:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fe300 x1862022294280960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768181 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1775768156.849804:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768156.849810:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775768156.849812:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00100000:0.0:1775768156.849816:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00000040:1.0:1775768156.849817:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768156.849819:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000008:00000001:0.0:1775768156.849825:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775768156.849826:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294280960, offset 0 00000100:00000001:0.0:1775768156.849831:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768156.849836:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00000010:0.0:1775768156.849838:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fd880. 00000020:00000040:0.0:1775768156.849843:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000400:00000200:1.0:1775768156.849845:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:0.0:1775768156.849853:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768156.849855:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768156.849858:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00000001:0.0:1775768156.849860:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768156.849863:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:1.0:1775768156.849863:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:0.0:1775768156.849870:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768156.849873:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 02000000:00000010:0.0:1775768156.849879:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220028000. 00000400:00000200:1.0:1775768156.849879:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768156.849886:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768156.849890:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225368:225368:256:4294967295] 192.168.202.156@tcp LPNI seq info [225364:225364:8:4294967295] 00000100:00000001:0.0:1775768156.849893:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768156.849893:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00100000:0.0:1775768156.849905:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fd880 x1862022294281088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775768156.849908:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768156.849932:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00000001:0.0:1775768156.849942:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000010:1.0:1775768156.849942:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000100:00000001:0.0:1775768156.849949:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775768156.849953:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000040:0.0:1775768156.849957:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007f8a5f3b] to pc [ptlrpcd_rcv+-1] req@ffff9942099fd880 x1862022294281088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1775768156.849958:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775768156.849977:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768156.849985:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000100:00000001:0.0:1775768156.849995:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1775768156.849996:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768156.850001:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000100:00080000:0.0:1775768156.850002:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1775768156.850008:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00080000:0.0:1775768156.850009:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775768156.850013:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768156.850018:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1775768156.850018:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000100:00000001:0.0:1775768156.850028:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:1.0:1775768156.850031:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000100:00080000:0.0:1775768156.850035:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5409 00000400:00000010:1.0:1775768156.850037:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000400:00000200:0.0:1775768156.850043:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768156.850045:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000400:00000200:0.0:1775768156.850049:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768156.850052:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000200:1.0:1775768156.850054:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fe300 x1862022294280960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768181 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775768156.850058:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768156.850064:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768156.850070:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768156.850080:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768156.850087:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768156.850090:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000100:00000001:1.0:1775768156.850090:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000020:00000001:0.0:1775768156.850093:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000040:1.0:1775768156.850096:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fe300 x1862022294280960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768181 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1775768156.850097:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:0.0:1775768156.850106:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:0.0:1775768156.850118:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000001:1.0:1775768156.850118:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768156.850126:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:0.0:1775768156.850130:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768156.850133:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768156.850134:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768156.850141:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000200:1.0:1775768156.850143:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768156.850150:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:1.0:1775768156.850151:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4300083005 00000100:00000010:1.0:1775768156.850171:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cbad80. 00000100:00000001:1.0:1775768156.850176:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775768156.850181:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4300083006 00000100:00000010:1.0:1775768156.850183:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba0c0. 00000100:00000001:1.0:1775768156.850186:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775768156.850189:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768156.850191:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768156.850193:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768156.850197:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768156/real 1775768156] req@ffff9942099fe300 x1862022294280960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768156.850203:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768156.850204:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768156.850206:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768156.850208:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768156.850209:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768156.850211:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768156.850213:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fe300 x1862022294280960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768156.850220:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fe300 x1862022294280960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768156.850224:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768156.850226:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768156.850227:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768156.850229:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768156.850232:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fe300 x1862022294280960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768156.850236:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:0.0:1775768156.850242:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00000040:1.0:1775768156.850243:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fe300 x1862022294280960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768181 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768156.850253:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775768156.850257:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768156.850260:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768156.850264:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1775768156.850265:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1775768156.850269:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:0.0:1775768156.850271:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768156.850271:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1775768156.850277:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768156.850278:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768156.850281:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768156.850282:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775768156.850285:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1775768156.850287:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000008:00000001:0.0:1775768156.850290:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768156.850294:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768156.850298:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000010:0.0:1775768156.850301:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ff480. 00000100:00000001:1.0:1775768156.850302:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000040:0.0:1775768156.850306:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000100:00000040:1.0:1775768156.850312:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fe300 x1862022294280960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768181 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1775768156.850313:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768156.850315:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768156.850324:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768156.850328:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768156.850332:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768156.850336:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294280960:192.168.202.156@tcp:250:kworker.0 02000000:00000010:0.0:1775768156.850340:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002b800. 00000100:00000001:1.0:1775768156.850342:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768156.850345:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1775768156.850347:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768156.850350:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fe300 x1862022294280960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768181 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768156.850357:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 00000100:00100000:0.0:1775768156.850358:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ff480 x1862022294281216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1775768156.850359:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768156.850363:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543fc00. 02000000:00000001:1.0:1775768156.850369:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768156.850372:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768156.850375:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000100:00000001:0.0:1775768156.850389:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768156.850394:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768156.850394:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768156.850399:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029400. 00000100:00000040:0.0:1775768156.850405:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e73dcd12] to pc [ptlrpcd_rcv+-1] req@ffff9942099ff480 x1862022294281216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1775768156.850405:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768156.850407:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768156.850412:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fe300. 00000100:00000001:1.0:1775768156.850416:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768156.850418:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768156.850424:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768156.850432:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768156.850434:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768156.850435:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768156.850444:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775768156.850444:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768156.850450:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1775768156.850453:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775768156.850454:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1775768156.850456:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768156.850458:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:1.0:1775768156.850464:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fd880 x1862022294281088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768156.850465:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768156.850471:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5409 00000400:00000200:0.0:1775768156.850482:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768156.850490:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768156.850497:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768156.850501:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768156.850510:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775768156.850510:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768156.850513:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768156.850515:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000001:1.0:1775768156.850519:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1775768156.850520:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 02000000:00000001:1.0:1775768156.850522:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1775768156.850525:0:13471:0:(genops.c:811:class_conn2export()) Process entered 02000000:00000001:1.0:1775768156.850527:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1775768156.850528:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:0.0:1775768156.850532:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00100000:1.0:1775768156.850533:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294281088:192.168.202.156@tcp:38:kworker.0 00000020:00000040:0.0:1775768156.850536:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000100:00000001:1.0:1775768156.850538:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000001:0.0:1775768156.850542:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 02000000:00000001:1.0:1775768156.850542:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768156.850547:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768156.850548:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 02000000:00000001:1.0:1775768156.850550:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768156.850553:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543fc00. 02000000:00000001:1.0:1775768156.850556:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768156.850558:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768156.850561:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1775768156.850562:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768156.850567:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000100:00000040:0.0:1775768156.850568:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768156.850573:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000010:1.0:1775768156.850575:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000020:00000040:0.0:1775768156.850576:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00000200:1.0:1775768156.850580:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294281088, portal 10 00000100:00080000:0.0:1775768156.850583:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775768156.850585:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768156.850586:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768156.850587:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641202816 : -112966068348800 : ffff9942099fd880) 02000000:00000001:0.0:1775768156.850593:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768156.850597:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1775768156.850598:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fd880 x1862022294281088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768181 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1775768156.850604:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768156.850611:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775768156.850618:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000100:00000001:1.0:1775768156.850619:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00000001:0.0:1775768156.850624:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768156.850625:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000100:00000001:0.0:1775768156.850633:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775768156.850634:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768156.850646:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294281088, offset 0 00000100:00000010:0.0:1775768156.850647:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fd180. 00000020:00000040:0.0:1775768156.850657:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00000400:00000200:1.0:1775768156.850664:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 02000000:00000001:0.0:1775768156.850665:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768156.850672:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768156.850678:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:1.0:1775768156.850678:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:0.0:1775768156.850684:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768156.850687:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768156.850687:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768156.850696:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000010:0.0:1775768156.850697:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002bc00. 00000400:00000200:1.0:1775768156.850700:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000100:00000001:0.0:1775768156.850706:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768156.850707:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768156.850718:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00100000:0.0:1775768156.850719:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fd180 x1862022294281344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775768156.850728:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225369:225369:256:4294967295] 192.168.202.156@tcp LPNI seq info [225365:225365:8:4294967295] 00000400:00000200:1.0:1775768156.850749:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768156.850753:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768156.850755:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768156.850763:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004ec16d17] to pc [ptlrpcd_rcv+-1] req@ffff9942099fd180 x1862022294281344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775768156.850771:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768156.850796:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00000001:0.0:1775768156.850799:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1775768156.850800:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000100:00080000:0.0:1775768156.850807:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:1.0:1775768156.850810:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768156.850815:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000100:00080000:0.0:1775768156.850818:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775768156.850821:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000800:00000200:1.1:1775768156.850822:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00080000:0.0:1775768156.850826:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:1.1:1775768156.850827:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000100:00000001:0.0:1775768156.850836:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000800:00000100:1.0:1775768156.850836:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768156.850842:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000100:00080000:0.0:1775768156.850846:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5409 00000400:00000200:1.0:1775768156.850847:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768156.850857:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768156.850859:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768156.850866:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000400:00000200:1.0:1775768156.850867:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000100:00080000:0.0:1775768156.850872:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000400:00000010:1.0:1775768156.850872:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768156.850875:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:0.0:1775768156.850880:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000200:1.0:1775768156.850881:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fd880 x1862022294281088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768181 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768156.850886:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768156.850894:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768156.850902:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768156.850912:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768156.850916:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000100:00000001:1.0:1775768156.850916:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000020:00000001:0.0:1775768156.850922:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768156.850927:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000100:00000040:1.0:1775768156.850927:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fd880 x1862022294281088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768181 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775768156.850937:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:0.0:1775768156.850947:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:0.0:1775768156.850958:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768156.850959:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768156.850960:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775768156.850964:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:0.0:1775768156.850965:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768156.850968:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:1.0:1775768156.850968:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1775768156.850978:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00000040:1.0:1775768156.850979:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000100:00080000:0.0:1775768156.850985:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000400:00000200:1.0:1775768156.850986:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768156.850994:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768156.850995:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300081683 02000000:00000001:0.0:1775768156.851001:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1775768156.851001:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300083005 00000100:00000001:1.0:1775768156.851007:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300081683 02000000:00000001:0.0:1775768156.851008:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768156.851009:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300083006 02000000:00000001:0.0:1775768156.851018:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768156.851018:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768156.851026:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775768156.851028:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1775768156.851029:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:0.0:1775768156.851034:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768156.851035:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1775768156.851037:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768156.851040:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fed80. 00000100:00100000:1.0:1775768156.851043:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768156/real 1775768156] req@ffff9942099fd880 x1862022294281088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1775768156.851044:0:13471:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:0.0:1775768156.851048:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768156.851050:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768156.851053:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768156.851055:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768156.851058:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768156.851065:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220029400. 00000100:00000001:0.0:1775768156.851070:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1775768156.851072:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768156.851075:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00100000:0.0:1775768156.851077:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fed80 x1862022294281472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:1.0:1775768156.851078:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768156.851087:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768156.851092:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768156.851097:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768156.851107:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fd880 x1862022294281088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768156.851119:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768156.851126:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768156.851138:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000c09b4d8] to pc [ptlrpcd_rcv+-1] req@ffff9942099fed80 x1862022294281472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768156.851139:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fd880 x1862022294281088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768156.851178:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768156.851184:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768156.851187:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768156.851187:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768156.851194:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768156.851196:0:13471:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5419) 00000100:00100000:1.0:1775768156.851199:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fd880 x1862022294281088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768156.851204:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768156.851207:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fd880 x1862022294281088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768181 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768156.851211:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768156.851216:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768156.851218:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768156.851219:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768156.851220:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768156.851222:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768156.851224:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768156.851226:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768156.851227:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768156.851230:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fd880 x1862022294281088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768181 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768156.851234:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294281088:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768156.851237:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768156.851239:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ff480 x1862022294281216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768156.851247:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768156.851248:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768156.851250:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768156.851252:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768156.851253:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768156.851254:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294281216:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768156.851256:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768156.851258:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768156.851258:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768156.851260:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768156.851262:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543e800. 02000000:00000001:1.0:1775768156.851264:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768156.851267:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768156.851270:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000400:00000010:1.0:1775768156.851273:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000200:1.0:1775768156.851275:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294281216, portal 10 00000100:00000001:1.0:1775768156.851277:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768156.851278:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641209984 : -112966068341632 : ffff9942099ff480) 00000100:00000040:1.0:1775768156.851282:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ff480 x1862022294281216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768181 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768156.851288:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768156.851290:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768156.851292:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768156.851294:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294281216, offset 0 00000400:00000200:1.0:1775768156.851298:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768156.851302:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768156.851306:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768156.851309:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768156.851311:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768156.851313:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768156.851315:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768156.851318:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225370:225370:256:4294967295] 192.168.202.156@tcp LPNI seq info [225366:225366:8:4294967295] 00000400:00000200:1.0:1775768156.851321:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768156.851326:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768156.851328:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768156.851331:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000800:00000200:1.0:1775768156.851333:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768156.851335:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775768156.851338:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768156.851339:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000100:1.0:1775768156.851341:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768156.851343:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768156.851344:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768156.851346:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768156.851348:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768156.851349:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768156.851351:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768156.851353:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ff480 x1862022294281216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768181 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768156.851357:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768156.851359:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ff480 x1862022294281216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768181 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768156.851363:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768156.851364:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768156.851365:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768156.851367:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:1.0:1775768156.851369:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768156.851371:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300081676 00000100:00000001:1.0:1775768156.851372:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300083005 00000100:00000001:1.0:1775768156.851373:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300081676 00000100:00000001:1.0:1775768156.851375:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300083006 00000100:00000001:1.0:1775768156.851387:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768156.851390:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768156.851391:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768156.851393:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768156/real 1775768156] req@ffff9942099ff480 x1862022294281216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768156.851399:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768156.851400:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768156.851401:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768156.851403:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768156.851404:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768156.851405:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768156.851406:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ff480 x1862022294281216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768156.851411:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ff480 x1862022294281216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768156.851415:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768156.851415:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768156.851416:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768156.851417:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768156.851419:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ff480 x1862022294281216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768156.851422:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768156.851424:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ff480 x1862022294281216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768181 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768156.851428:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768156.851429:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768156.851430:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768156.851431:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768156.851432:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768156.851434:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768156.851435:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768156.851437:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768156.851438:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768156.851440:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ff480 x1862022294281216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768181 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768156.851445:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294281216:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768156.851447:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768156.851448:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768156.851449:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fd880 x1862022294281088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768181 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768156.851453:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768156.851454:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768156.851455:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543fc00. 02000000:00000001:1.0:1775768156.851456:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768156.851457:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768156.851458:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:1.0:1775768156.851459:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768156.851460:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028000. 02000000:00000001:1.0:1775768156.851462:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768156.851463:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768156.851465:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fd880. 00000100:00000001:1.0:1775768156.851467:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768156.851468:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768156.851469:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768156.851472:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ff480 x1862022294281216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768181 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768156.851478:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768156.851479:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768156.851480:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543e800. 02000000:00000001:1.0:1775768156.851482:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768156.851483:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768156.851484:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:1.0:1775768156.851486:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768156.851487:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002b800. 02000000:00000001:1.0:1775768156.851489:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768156.851490:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768156.851491:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ff480. 00000100:00000001:1.0:1775768156.851494:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768156.851495:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768156.851497:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768156.851500:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768156.851502:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768156.851504:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768156.851505:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768156.851507:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768156.851511:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fd180 x1862022294281344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768156.851517:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768156.851518:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768156.851520:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768156.851522:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768156.851524:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768156.851526:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294281344:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768156.851529:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768156.851531:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768156.851532:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768156.851533:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768156.851535:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543e800. 02000000:00000001:1.0:1775768156.851536:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768156.851539:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768156.851540:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000400:00000010:1.0:1775768156.851542:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000200:1.0:1775768156.851544:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294281344, portal 4 00000100:00000001:1.0:1775768156.851545:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768156.851546:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641201024 : -112966068350592 : ffff9942099fd180) 00000100:00000040:1.0:1775768156.851549:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fd180 x1862022294281344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768181 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768156.851556:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768156.851558:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768156.851559:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768156.851560:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294281344, offset 0 00000400:00000200:1.0:1775768156.851562:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768156.851565:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768156.851567:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768156.851568:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768156.851570:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768156.851572:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768156.851573:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768156.851575:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225371:225371:256:4294967295] 192.168.202.156@tcp LPNI seq info [225367:225367:8:4294967295] 00000400:00000200:1.0:1775768156.851577:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768156.851581:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768156.851583:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768156.851585:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000800:00000200:1.0:1775768156.851588:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768156.851589:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775768156.851591:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768156.851598:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000100:1.0:1775768156.851600:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768156.851602:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768156.851603:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768156.851605:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768156.851607:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768156.851608:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768156.851610:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768156.851612:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fd180 x1862022294281344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768181 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768156.851616:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768156.851618:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fd180 x1862022294281344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768181 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768156.851625:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768156.851626:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768156.851628:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768156.851630:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:1.0:1775768156.851632:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768156.851633:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300081699 00000100:00000001:1.0:1775768156.851635:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300083005 00000100:00000001:1.0:1775768156.851636:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300081699 00000100:00000001:1.0:1775768156.851637:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300083006 00000100:00000001:1.0:1775768156.851638:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768156.851639:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768156.851640:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768156.851642:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768156/real 1775768156] req@ffff9942099fd180 x1862022294281344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768156.851646:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768156.851647:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768156.851648:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768156.851650:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768156.851651:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768156.851652:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768156.851654:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fd180 x1862022294281344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768156.851658:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fd180 x1862022294281344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768156.851662:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768156.851662:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768156.851663:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768156.851664:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768156.851666:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fd180 x1862022294281344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768156.851670:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768156.851672:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fd180 x1862022294281344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768181 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768156.851675:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768156.851676:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768156.851677:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768156.851678:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768156.851679:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768156.851681:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768156.851682:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768156.851684:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768156.851685:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768156.851687:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fd180 x1862022294281344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768181 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768156.851691:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294281344:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768156.851693:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768156.851695:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fed80 x1862022294281472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768156.851699:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768156.851700:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768156.851701:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768156.851702:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768156.851703:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768156.851704:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294281472:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768156.851706:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768156.851707:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768156.851708:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768156.851709:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768156.851711:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543fc00. 02000000:00000001:1.0:1775768156.851712:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768156.851714:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768156.851715:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000400:00000010:1.0:1775768156.851717:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000200:1.0:1775768156.851718:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294281472, portal 4 00000100:00000001:1.0:1775768156.851719:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768156.851720:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641208192 : -112966068343424 : ffff9942099fed80) 00000100:00000040:1.0:1775768156.851724:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fed80 x1862022294281472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768181 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768156.851728:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768156.851729:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768156.851730:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768156.851731:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294281472, offset 0 00000400:00000200:1.0:1775768156.851733:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768156.851735:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768156.851736:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768156.851738:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768156.851739:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768156.851740:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768156.851742:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768156.851744:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225372:225372:256:4294967295] 192.168.202.156@tcp LPNI seq info [225368:225368:8:4294967295] 00000400:00000200:1.0:1775768156.851746:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768156.851749:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768156.851752:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768156.851753:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000800:00000200:1.0:1775768156.851755:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768156.851756:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775768156.851759:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768156.851760:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000100:1.0:1775768156.851762:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768156.851763:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768156.851765:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768156.851767:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768156.851768:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768156.851769:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768156.851770:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768156.851772:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fed80 x1862022294281472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768181 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768156.851776:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768156.851778:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fed80 x1862022294281472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768181 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768156.851781:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768156.851782:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768156.851783:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768156.851784:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:1.0:1775768156.851786:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768156.851787:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300081702 00000100:00000001:1.0:1775768156.851788:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300083005 00000100:00000001:1.0:1775768156.851789:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300081702 00000100:00000001:1.0:1775768156.851791:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300083006 00000100:00000001:1.0:1775768156.851792:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768156.851793:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768156.851794:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768156.851795:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768156/real 1775768156] req@ffff9942099fed80 x1862022294281472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768156.851799:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768156.851800:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768156.851801:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768156.851802:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768156.851802:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768156.851804:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768156.851805:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fed80 x1862022294281472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768156.851810:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fed80 x1862022294281472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768156.851813:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768156.851814:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768156.851815:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768156.851815:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768156.851817:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fed80 x1862022294281472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768156.851820:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768156.851822:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fed80 x1862022294281472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768181 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768156.851825:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768156.851827:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768156.851827:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768156.851828:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768156.851829:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768156.851831:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768156.851832:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768156.851834:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768156.851835:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768156.851836:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fed80 x1862022294281472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768181 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768156.851840:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294281472:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768156.851842:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768156.851844:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768156.851845:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fd180 x1862022294281344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768181 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768156.851849:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768156.851850:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768156.851851:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543e800. 02000000:00000001:1.0:1775768156.851852:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768156.851853:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768156.851853:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:1.0:1775768156.851854:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768156.851855:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002bc00. 02000000:00000001:1.0:1775768156.851857:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768156.851858:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768156.851858:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fd180. 00000100:00000001:1.0:1775768156.851860:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768156.851861:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768156.851862:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768156.851863:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fed80 x1862022294281472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768181 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768156.851867:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768156.851867:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768156.851868:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543fc00. 02000000:00000001:1.0:1775768156.851869:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768156.851869:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768156.851870:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:1.0:1775768156.851871:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768156.851871:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029400. 02000000:00000001:1.0:1775768156.851872:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768156.851873:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768156.851874:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fed80. 00000100:00000001:1.0:1775768156.851875:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768156.851875:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768156.851876:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768156.851878:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768156.851879:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768156.851880:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768156.851881:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768156.913281:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768156.913287:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768156.913293:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768156.913297:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768156.913299:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768156.913303:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768156.913311:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768156.913313:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768156.913315:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768156.913317:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768156.913322:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768156.913323:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768156.913324:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768156.913326:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768156.913328:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768156.913329:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768156.913331:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768156.913333:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768156.913336:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768156.913338:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768156.913340:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768156.913341:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768156.913343:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768156.913344:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768156.913345:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768156.913347:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768156.913350:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768156.913352:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768156.913353:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768156.913355:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768156.913357:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768156.913358:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768156.913359:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768156.913361:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768156.913442:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768156.913447:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768156.913450:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768156.977294:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768156.977303:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768156.977305:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768156.977311:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768156.977315:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768156.977319:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768156.977321:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768156.977324:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768156.977326:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768157.873251:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768157.873251:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768157.873252:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768157.873258:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768157.873259:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768157.873259:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768157.873261:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768157.873262:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768157.873262:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768157.873268:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775768157.873268:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:2.0:1775768157.873269:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768157.873275:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768157.873275:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768157.873275:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768157.873279:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768157.873279:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768157.873280:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768157.873284:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768157.873287:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768157.873289:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768157.873290:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768157.873293:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768157.873296:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768157.873296:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768157.873298:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768157.873299:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768157.937263:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768157.937267:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768157.937271:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768157.937272:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768157.937274:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768157.937275:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768157.937277:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768157.937278:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768157.937280:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768157.937281:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768157.937352:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768157.937359:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768157.937365:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768158.001259:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768158.001264:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768158.001265:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768158.001269:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768158.001272:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768158.001275:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768158.001276:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768158.001278:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768158.001280:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768158.193295:0:15036:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4300083005, at 4300083008 00000100:00000040:1.0:1775768158.193295:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4300083006, at 4300083008 00000100:00000010:1.0:1775768158.193315:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cba0c0. 00000100:00000010:0.0:1775768158.193324:0:15036:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cbad80. 00000100:00000001:1.0:1775768158.897285:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768158.897285:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768158.897285:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768158.897292:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768158.897293:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768158.897293:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768158.897295:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768158.897296:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768158.897297:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768158.897303:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:2.0:1775768158.897304:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775768158.897304:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768158.897309:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768158.897310:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768158.897310:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768158.897313:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768158.897314:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768158.897315:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768158.897316:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768158.897319:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768158.897323:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768158.897323:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768158.897326:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768158.897329:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768158.897330:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768158.897333:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768158.897337:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768158.961285:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768158.961292:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768158.961299:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768158.961300:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768158.961304:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768158.961305:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768158.961309:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768158.961310:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768158.961314:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768158.961315:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768158.961372:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768158.961379:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768158.961382:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768159.025279:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768159.025285:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768159.025287:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768159.025292:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768159.025295:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768159.025299:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768159.025301:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768159.025304:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768159.025306:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768159.921225:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768159.921225:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768159.921232:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768159.921232:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768159.921234:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768159.921235:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768159.921240:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775768159.921241:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768159.921245:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768159.921246:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768159.921249:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768159.921251:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768159.921253:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768159.921256:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768159.921256:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768159.921260:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768159.921262:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768159.921265:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768159.921270:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768159.921277:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768159.921279:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768159.921285:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768159.921290:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768159.921294:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768159.921296:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768159.921299:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768159.921302:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768159.985282:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768159.985288:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768159.985292:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768159.985293:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768159.985296:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768159.985297:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768159.985299:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768159.985300:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768159.985302:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768159.985303:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768159.985371:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768159.985374:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768159.985377:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768160.049306:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768160.049314:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768160.049317:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768160.049324:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768160.049330:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768160.049335:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768160.049338:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768160.049341:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768160.049356:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768160.945253:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768160.945253:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768160.945260:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768160.945261:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768160.945261:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768160.945265:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768160.945267:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768160.945269:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:2.0:1775768160.945273:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768160.945274:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768160.945274:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768160.945286:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768160.945286:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1775768160.945287:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775768160.945291:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768160.945294:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768160.945299:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768160.945299:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768160.945306:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768160.945308:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768160.945308:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768160.945312:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768160.945312:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768160.945321:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768160.945322:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768160.945326:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768160.945330:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768161.009256:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768161.009260:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768161.009265:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768161.009266:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768161.009268:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768161.009269:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768161.009271:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768161.009272:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768161.009274:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768161.009275:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768161.009330:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768161.009333:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768161.009357:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768161.073215:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768161.073221:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768161.073222:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768161.073226:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768161.073228:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768161.073231:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768161.073232:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768161.073234:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768161.073236:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768161.969207:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768161.969212:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768161.969214:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768161.969219:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775768161.969221:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768161.969224:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1775768161.969225:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1775768161.969229:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1775768161.969230:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775768161.969234:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1775768161.969234:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1775768161.969237:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775768161.969238:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768161.969239:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768161.969241:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775768161.969244:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768161.969246:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:0.0:1775768161.969248:0:13471:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5414 00000100:00000001:2.0:1775768161.969248:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768161.969251:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768161.969254:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768161.969254:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000400:00000200:0.0:1775768161.969259:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00100000:1.0:1775768161.969259:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775768161.969259:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768161.969263:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768161.969264:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000400:00000200:0.0:1775768161.969267:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:2.0:1775768161.969267:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768161.969268:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768161.969271:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768161.969275:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768161.969277:0:13471:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1775768161.969278:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000040:0.0:1775768161.969282:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768161.969282:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768161.969286:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768161.969287:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768161.969290:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768161.969292:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768161.969295:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768161.969296:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:0.0:1775768161.969298:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768161.969300:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:0.0:1775768161.969302:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:0.0:1775768161.969304:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:0.0:1775768161.969306:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768161.969307:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768161.969309:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768161.969310:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768161.969313:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:0.0:1775768161.969314:0:13471:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768161.969316:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768161.969318:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768161.969320:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768161.969321:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768161.969323:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775768161.969324:0:13471:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768161.969341:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fc380. 00000020:00000040:0.0:1775768161.969344:0:13471:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:0.0:1775768161.969347:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768161.969348:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768161.969350:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768161.969352:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768161.969355:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768161.969360:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220028400. 00000100:00000001:0.0:1775768161.969365:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768161.969371:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fc380 x1862022294281600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768161.969383:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768161.969386:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768161.969389:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002957419c] to pc [ptlrpcd_rcv+-1] req@ffff9942099fc380 x1862022294281600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768161.969402:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768161.969404:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768161.969407:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775768161.969407:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768161.969408:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768161.969410:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768161.969412:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5414 00000400:00000200:0.0:1775768161.969414:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768161.969416:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768161.969417:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768161.969419:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768161.969420:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768161.969422:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768161.969423:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768161.969425:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768161.969425:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:0.0:1775768161.969426:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768161.969428:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:0.0:1775768161.969429:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:0.0:1775768161.969431:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:0.0:1775768161.969432:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768161.969434:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768161.969435:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768161.969436:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775768161.969437:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768161.969440:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00000001:1.0:1775768161.969440:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1775768161.969443:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775768161.969444:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1775768161.969445:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768161.969448:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768161.969451:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768161.969453:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768161.969454:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fc380 x1862022294281600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768161.969455:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775768161.969459:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1775768161.969463:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000008:00000001:0.0:1775768161.969465:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768161.969465:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768161.969469:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768161.969471:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000010:0.0:1775768161.969473:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fea00. 02000000:00000001:1.0:1775768161.969474:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:0.0:1775768161.969477:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:1.0:1775768161.969479:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768161.969481:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768161.969483:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768161.969485:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294281600:192.168.202.156@tcp:250:kworker.0 00000100:00000001:0.0:1775768161.969486:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768161.969488:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768161.969492:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768161.969497:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000010:0.0:1775768161.969499:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002a800. 02000000:00000001:1.0:1775768161.969501:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1775768161.969504:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768161.969504:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768161.969510:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:0.0:1775768161.969513:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fea00 x1862022294281728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000010:1.0:1775768161.969517:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543fc00. 02000000:00000001:1.0:1775768161.969521:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768161.969529:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768161.969533:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000100:00000001:0.0:1775768161.969534:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768161.969538:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775768161.969542:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000040:0.0:1775768161.969543:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008d4eaacc] to pc [ptlrpcd_rcv+-1] req@ffff9942099fea00 x1862022294281728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768161.969551:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294281600, portal 25 00000100:00000001:0.0:1775768161.969554:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768161.969555:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768161.969557:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641197440 : -112966068354176 : ffff9942099fc380) 00000100:00080000:0.0:1775768161.969558:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768161.969564:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775768161.969565:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768161.969569:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:1.0:1775768161.969569:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fc380 x1862022294281600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768186 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768161.969576:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768161.969580:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000100:00000001:0.0:1775768161.969581:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000400:00000010:1.0:1775768161.969583:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00080000:0.0:1775768161.969586:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5414 00000100:00000200:1.0:1775768161.969588:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294281600, offset 0 00000400:00000200:0.0:1775768161.969592:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768161.969597:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000400:00000200:1.0:1775768161.969599:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00080000:0.0:1775768161.969601:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768161.969606:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000200:1.0:1775768161.969609:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768161.969610:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768161.969613:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768161.969615:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000200:1.0:1775768161.969616:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000020:00000001:0.0:1775768161.969617:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768161.969619:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:0.0:1775768161.969621:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768161.969622:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000400:00000200:1.0:1775768161.969623:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:0.0:1775768161.969626:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:0.0:1775768161.969628:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000400:00000200:1.0:1775768161.969629:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000100:00000040:0.0:1775768161.969632:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000200:1.0:1775768161.969635:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1775768161.969636:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768161.969640:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000200:1.0:1775768161.969641:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00000001:0.0:1775768161.969642:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768161.969645:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000400:00000200:1.0:1775768161.969647:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225373:225373:256:4294967295] 192.168.202.156@tcp LPNI seq info [225369:225369:8:4294967295] 00000100:00080000:0.0:1775768161.969651:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000400:00000200:1.0:1775768161.969654:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768161.969657:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768161.969660:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768161.969665:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1775768161.969665:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 02000000:00000001:0.0:1775768161.969670:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768161.969673:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000800:00000200:1.0:1775768161.969674:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000008:00100000:0.0:1775768161.969677:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000010:1.0:1775768161.969678:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000008:00000001:0.0:1775768161.969682:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768161.969685:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775768161.969688:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000010:0.0:1775768161.969690:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fdc00. 00000800:00000200:1.0:1775768161.969692:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000020:00000040:0.0:1775768161.969694:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:0.0:1775768161.969699:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768161.969700:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768161.969704:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000800:00000200:1.1:1775768161.969704:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 02000000:00000001:0.0:1775768161.969707:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:1.1:1775768161.969708:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 02000000:00000001:0.0:1775768161.969709:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1775768161.969713:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 02000000:00000010:0.0:1775768161.969716:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220029800. 00000800:00000010:1.0:1775768161.969716:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000100:00000001:0.0:1775768161.969722:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768161.969724:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00100000:0.0:1775768161.969729:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fdc00 x1862022294281856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775768161.969732:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768161.969740:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768161.969743:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768161.969747:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1775768161.969751:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768161.969754:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775768161.969754:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fc380 x1862022294281600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768186 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775768161.969760:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001372d9ae] to pc [ptlrpcd_rcv+-1] req@ffff9942099fdc00 x1862022294281856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768161.969778:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768161.969779:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00080000:0.0:1775768161.969780:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768161.969783:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775768161.969785:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768161.969786:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:1.0:1775768161.969787:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fc380 x1862022294281600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768186 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768161.969788:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768161.969792:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5414 00000400:00000200:0.0:1775768161.969799:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768161.969801:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768161.969803:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:1.0:1775768161.969804:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:0.0:1775768161.969805:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768161.969808:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768161.969810:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775768161.969811:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768161.969813:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768161.969817:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000200:1.0:1775768161.969817:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:0.0:1775768161.969822:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:1.0:1775768161.969822:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4300088125 00000020:00000040:0.0:1775768161.969823:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:0.0:1775768161.969826:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768161.969829:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000100:00000010:1.0:1775768161.969829:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba0c0. 00000020:00000001:0.0:1775768161.969833:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:0.0:1775768161.969835:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:1.0:1775768161.969837:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000040:0.0:1775768161.969840:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768161.969843:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775768161.969844:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4300088125 00000100:00000040:0.0:1775768161.969847:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000010:1.0:1775768161.969847:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba2a0. 00000100:00000001:0.0:1775768161.969851:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:1.0:1775768161.969852:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000020:00000040:0.0:1775768161.969857:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00000001:1.0:1775768161.969857:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768161.969861:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768161.969863:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775768161.969864:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1775768161.969867:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768161.969870:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768161/real 1775768161] req@ffff9942099fc380 x1862022294281600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1775768161.969871:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768161.969875:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768161.969879:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1775768161.969879:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1775768161.969880:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000001:1.0:1775768161.969881:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000008:00100000:0.0:1775768161.969883:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000400:00000010:1.0:1775768161.969886:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768161.969890:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000008:00000001:0.0:1775768161.969892:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775768161.969892:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768161.969894:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1775768161.969895:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768161.969899:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fce00. 00000100:00000200:1.0:1775768161.969899:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fc380 x1862022294281600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1775768161.969903:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:0.0:1775768161.969907:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768161.969909:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768161.969912:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768161.969914:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768161.969915:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775768161.969916:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fc380 x1862022294281600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1775768161.969918:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220029400. 00000100:00000001:0.0:1775768161.969923:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768161.969924:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768161.969926:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768161.969928:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00100000:0.0:1775768161.969930:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fce00 x1862022294281984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775768161.969930:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768161.969939:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00100000:1.0:1775768161.969939:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fc380 x1862022294281600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768161.969942:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768161.969948:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000060e5e9d1] to pc [ptlrpcd_rcv+-1] req@ffff9942099fce00 x1862022294281984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768161.969955:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768161.969960:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768161.969962:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1775768161.969963:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fc380 x1862022294281600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768186 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768161.969967:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775768161.969970:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768161.969972:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768161.969978:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775768161.969979:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768161.969982:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1775768161.969984:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5414 00000100:00000001:1.0:1775768161.969984:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768161.969986:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768161.969989:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768161.969990:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000400:00000200:0.0:1775768161.969992:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00080000:1.0:1775768161.969994:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000400:00000200:0.0:1775768161.969997:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768161.970001:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768161.970001:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768161.970005:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768161.970010:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fc380 x1862022294281600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768186 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775768161.970014:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00100000:1.0:1775768161.970015:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294281600:192.168.202.156@tcp:250:kworker.0 00000100:00000001:0.0:1775768161.970020:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768161.970024:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768161.970027:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:1.0:1775768161.970028:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1775768161.970029:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768161.970030:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:0.0:1775768161.970033:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:1.0:1775768161.970033:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000020:00000040:0.0:1775768161.970034:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:0.0:1775768161.970036:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:0.0:1775768161.970039:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:1.0:1775768161.970040:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fc380 x1862022294281600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768186 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775768161.970042:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768161.970045:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768161.970048:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768161.970051:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768161.970054:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:0.0:1775768161.970058:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768161.970061:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768161.970063:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1775768161.970064:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768161.970065:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768161.970067:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1775768161.970068:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768161.970070:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775768161.970072:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 02000000:00000010:1.0:1775768161.970072:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543fc00. 00000008:00000001:0.0:1775768161.970076:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768161.970078:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768161.970078:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000010:0.0:1775768161.970080:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fca80. 00000020:00000001:1.0:1775768161.970081:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768161.970082:0:13471:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000040:1.0:1775768161.970084:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 02000000:00000001:0.0:1775768161.970085:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768161.970087:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768161.970089:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000020:00000001:1.0:1775768161.970089:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000001:0.0:1775768161.970092:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768161.970093:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775768161.970093:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028400. 02000000:00000010:0.0:1775768161.970097:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002bc00. 02000000:00000001:1.0:1775768161.970098:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:0.0:1775768161.970100:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768161.970101:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768161.970104:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fc380. 00000100:00100000:0.0:1775768161.970106:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fca80 x1862022294282112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775768161.970107:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768161.970110:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768161.970115:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768161.970116:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768161.970119:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768161.970121:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000046a3ca2f] to pc [ptlrpcd_rcv+-1] req@ffff9942099fca80 x1862022294282112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768161.970123:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768161.970126:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768161.970131:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768161.970131:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768161.970133:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1775768161.970135:0:13471:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5424) 00000100:00000001:1.0:1775768161.970135:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768161.970142:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fea00 x1862022294281728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768161.970150:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768161.970151:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768161.970153:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768161.970154:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768161.970171:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768161.970175:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294281728:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768161.970178:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768161.970179:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768161.970180:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768161.970182:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768161.970184:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543fc00. 02000000:00000001:1.0:1775768161.970185:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768161.970188:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768161.970189:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000400:00000010:1.0:1775768161.970191:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000200:1.0:1775768161.970193:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294281728, portal 10 00000100:00000001:1.0:1775768161.970195:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768161.970196:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641207296 : -112966068344320 : ffff9942099fea00) 00000100:00000040:1.0:1775768161.970210:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fea00 x1862022294281728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768186 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768161.970217:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768161.970219:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768161.970222:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768161.970224:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294281728, offset 0 00000400:00000200:1.0:1775768161.970228:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768161.970232:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768161.970235:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768161.970238:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768161.970240:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768161.970243:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768161.970245:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768161.970248:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225374:225374:256:4294967295] 192.168.202.156@tcp LPNI seq info [225370:225370:8:4294967295] 00000400:00000200:1.0:1775768161.970252:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768161.970258:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768161.970262:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768161.970266:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000800:00000200:1.0:1775768161.970270:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768161.970272:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775768161.970276:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768161.970278:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000100:1.0:1775768161.970282:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768161.970284:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768161.970286:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768161.970289:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768161.970292:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768161.970294:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768161.970296:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768161.970299:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fea00 x1862022294281728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768186 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768161.970304:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768161.970306:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fea00 x1862022294281728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768186 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768161.970310:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768161.970311:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768161.970314:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768161.970316:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:1.0:1775768161.970318:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768161.970320:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300086803 00000100:00000001:1.0:1775768161.970321:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300088125 00000100:00000001:1.0:1775768161.970323:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300086803 00000100:00000001:1.0:1775768161.970324:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300088125 00000100:00000001:1.0:1775768161.970325:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768161.970337:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768161.970340:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768161.970343:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768161/real 1775768161] req@ffff9942099fea00 x1862022294281728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768161.970350:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768161.970351:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768161.970352:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768161.970354:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768161.970355:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768161.970356:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768161.970358:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fea00 x1862022294281728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768161.970363:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fea00 x1862022294281728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768161.970367:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768161.970367:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768161.970368:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768161.970369:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768161.970371:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fea00 x1862022294281728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768161.970374:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768161.970377:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fea00 x1862022294281728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768186 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768161.970382:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768161.970384:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768161.970385:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768161.970386:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768161.970388:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768161.970391:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768161.970393:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768161.970395:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768161.970396:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768161.970399:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fea00 x1862022294281728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768186 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768161.970403:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294281728:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768161.970405:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768161.970407:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fdc00 x1862022294281856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768161.970417:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768161.970419:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768161.970420:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768161.970421:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768161.970423:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768161.970424:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294281856:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768161.970426:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768161.970427:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768161.970428:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768161.970429:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768161.970432:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543e800. 02000000:00000001:1.0:1775768161.970433:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768161.970435:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768161.970436:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000400:00000010:1.0:1775768161.970438:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000200:1.0:1775768161.970441:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294281856, portal 10 00000100:00000001:1.0:1775768161.970443:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768161.970444:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641203712 : -112966068347904 : ffff9942099fdc00) 00000100:00000040:1.0:1775768161.970449:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fdc00 x1862022294281856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768186 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768161.970453:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768161.970454:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768161.970455:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768161.970457:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294281856, offset 0 00000400:00000200:1.0:1775768161.970459:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768161.970461:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768161.970463:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768161.970465:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768161.970466:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768161.970468:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768161.970470:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768161.970473:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225375:225375:256:4294967295] 192.168.202.156@tcp LPNI seq info [225371:225371:8:4294967295] 00000400:00000200:1.0:1775768161.970477:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768161.970481:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768161.970484:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768161.970487:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000800:00000200:1.0:1775768161.970490:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768161.970493:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775768161.970496:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768161.970498:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000100:1.0:1775768161.970502:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768161.970504:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768161.970506:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768161.970509:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768161.970512:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768161.970513:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768161.970515:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768161.970518:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fdc00 x1862022294281856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768186 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768161.970526:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768161.970529:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fdc00 x1862022294281856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768186 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768161.970535:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768161.970536:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768161.970540:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768161.970543:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:1.0:1775768161.970546:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768161.970547:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300086795 00000100:00000001:1.0:1775768161.970550:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300088125 00000100:00000001:1.0:1775768161.970551:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300086795 00000100:00000001:1.0:1775768161.970554:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300088125 00000100:00000001:1.0:1775768161.970556:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768161.970558:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768161.970560:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768161.970563:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768161/real 1775768161] req@ffff9942099fdc00 x1862022294281856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768161.970571:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768161.970572:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768161.970573:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768161.970575:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768161.970576:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768161.970577:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768161.970579:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fdc00 x1862022294281856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768161.970584:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fdc00 x1862022294281856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768161.970588:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768161.970589:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768161.970591:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768161.970592:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768161.970595:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fdc00 x1862022294281856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768161.970600:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768161.970602:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fdc00 x1862022294281856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768186 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768161.970606:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768161.970608:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768161.970609:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768161.970610:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768161.970610:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768161.970612:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768161.970613:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768161.970615:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768161.970617:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768161.970619:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fdc00 x1862022294281856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768186 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768161.970625:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294281856:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768161.970629:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768161.970631:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fce00 x1862022294281984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768161.970635:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768161.970636:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768161.970638:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768161.970639:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768161.970640:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768161.970641:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294281984:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768161.970643:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768161.970644:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768161.970645:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768161.970646:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768161.970649:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543dc00. 02000000:00000001:1.0:1775768161.970651:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768161.970654:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768161.970657:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000400:00000010:1.0:1775768161.970659:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000200:1.0:1775768161.970661:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294281984, portal 4 00000100:00000001:1.0:1775768161.970662:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768161.970663:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641200128 : -112966068351488 : ffff9942099fce00) 00000100:00000040:1.0:1775768161.970666:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fce00 x1862022294281984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768186 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768161.970670:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768161.970672:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768161.970673:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768161.970674:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294281984, offset 0 00000400:00000200:1.0:1775768161.970677:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768161.970681:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768161.970684:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768161.970687:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768161.970688:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768161.970690:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768161.970691:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768161.970693:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225376:225376:256:4294967295] 192.168.202.156@tcp LPNI seq info [225372:225372:8:4294967295] 00000400:00000200:1.0:1775768161.970695:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768161.970699:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768161.970702:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768161.970704:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000800:00000200:1.0:1775768161.970707:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768161.970710:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775768161.970713:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768161.970715:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000100:1.0:1775768161.970718:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768161.970719:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768161.970721:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768161.970723:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768161.970724:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768161.970725:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768161.970727:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768161.970729:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fce00 x1862022294281984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768186 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768161.970733:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768161.970736:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fce00 x1862022294281984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768186 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768161.970742:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768161.970744:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768161.970746:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768161.970748:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:1.0:1775768161.970750:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768161.970751:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300086818 00000100:00000001:1.0:1775768161.970753:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300088125 00000100:00000001:1.0:1775768161.970754:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300086818 00000100:00000001:1.0:1775768161.970755:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300088125 00000100:00000001:1.0:1775768161.970757:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768161.970758:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768161.970759:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768161.970761:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768161/real 1775768161] req@ffff9942099fce00 x1862022294281984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768161.970766:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768161.970767:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768161.970768:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768161.970770:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768161.970772:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768161.970773:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768161.970776:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fce00 x1862022294281984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768161.970782:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fce00 x1862022294281984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768161.970787:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768161.970788:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768161.970790:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768161.970791:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768161.970794:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fce00 x1862022294281984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768161.970798:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768161.970800:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fce00 x1862022294281984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768186 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768161.970805:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768161.970807:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768161.970808:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768161.970810:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768161.970812:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768161.970814:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768161.970815:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768161.970817:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768161.970819:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768161.970821:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fce00 x1862022294281984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768186 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768161.970825:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294281984:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768161.970827:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768161.970830:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fca80 x1862022294282112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768161.970836:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768161.970838:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768161.970839:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768161.970840:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768161.970841:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768161.970843:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294282112:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768161.970845:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768161.970846:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768161.970847:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768161.970848:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768161.970850:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543d800. 02000000:00000001:1.0:1775768161.970851:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768161.970853:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768161.970855:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000400:00000010:1.0:1775768161.970857:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000200:1.0:1775768161.970859:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294282112, portal 4 00000100:00000001:1.0:1775768161.970861:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768161.970863:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641199232 : -112966068352384 : ffff9942099fca80) 00000100:00000040:1.0:1775768161.970868:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fca80 x1862022294282112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768186 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768161.970874:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768161.970876:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768161.970877:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000100:00000200:1.0:1775768161.970879:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294282112, offset 0 00000400:00000200:1.0:1775768161.970881:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768161.970883:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768161.970885:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768161.970888:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768161.970890:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768161.970892:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768161.970895:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768161.970897:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225377:225377:256:4294967295] 192.168.202.156@tcp LPNI seq info [225373:225373:8:4294967295] 00000400:00000200:1.0:1775768161.970899:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768161.970903:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768161.970906:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768161.970909:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000800:00000200:1.0:1775768161.970912:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768161.970914:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775768161.970918:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768161.970919:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000100:1.0:1775768161.970921:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768161.970922:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:1.0:1775768161.970924:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768161.970926:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768161.970928:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:1.0:1775768161.970929:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:1.0:1775768161.970930:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768161.970932:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fca80 x1862022294282112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768186 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768161.970939:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768161.970942:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fca80 x1862022294282112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768186 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768161.970948:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768161.970950:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768161.970952:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768161.970953:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:1.0:1775768161.970956:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768161.970957:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300086821 00000100:00000001:1.0:1775768161.970958:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300088125 00000100:00000001:1.0:1775768161.970959:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300086821 00000100:00000001:1.0:1775768161.970961:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300088125 00000100:00000001:1.0:1775768161.970963:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768161.970964:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768161.970966:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768161.970969:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768161/real 1775768161] req@ffff9942099fca80 x1862022294282112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768161.970976:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768161.970977:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768161.970978:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:1.0:1775768161.970981:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:1.0:1775768161.970982:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:1.0:1775768161.970983:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768161.970985:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fca80 x1862022294282112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768161.970990:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fca80 x1862022294282112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768161.970999:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768161.971001:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768161.971002:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768161.971004:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768161.971007:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fca80 x1862022294282112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768161.971011:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768161.971013:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fca80 x1862022294282112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768186 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768161.971017:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768161.971018:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768161.971019:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768161.971020:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768161.971022:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768161.971025:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768161.971027:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768161.971029:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768161.971031:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768161.971035:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fca80 x1862022294282112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768186 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768161.971042:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294282112:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768161.971045:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768161.971047:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768161.971050:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fea00 x1862022294281728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768186 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768161.971056:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768161.971057:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768161.971059:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543fc00. 02000000:00000001:1.0:1775768161.971061:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768161.971062:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768161.971064:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:1.0:1775768161.971065:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768161.971067:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002a800. 02000000:00000001:1.0:1775768161.971069:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768161.971070:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768161.971072:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fea00. 00000100:00000001:1.0:1775768161.971074:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768161.971076:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768161.971077:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768161.971080:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fdc00 x1862022294281856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768186 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768161.971086:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768161.971087:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768161.971089:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543e800. 02000000:00000001:1.0:1775768161.971091:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768161.971093:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768161.971094:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:1.0:1775768161.971095:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768161.971097:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029800. 02000000:00000001:1.0:1775768161.971099:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768161.971100:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768161.971101:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fdc00. 00000100:00000001:1.0:1775768161.971103:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768161.971104:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768161.971105:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768161.971108:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fce00 x1862022294281984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768186 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768161.971114:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768161.971116:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768161.971117:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543dc00. 02000000:00000001:1.0:1775768161.971119:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768161.971120:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768161.971121:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:1.0:1775768161.971123:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768161.971124:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029400. 02000000:00000001:1.0:1775768161.971126:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768161.971127:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768161.971128:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fce00. 00000100:00000001:1.0:1775768161.971129:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768161.971130:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768161.971131:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768161.971133:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fca80 x1862022294282112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768186 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768161.971139:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768161.971140:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768161.971142:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543d800. 02000000:00000001:1.0:1775768161.971144:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768161.971145:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768161.971146:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:1.0:1775768161.971147:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768161.971149:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002bc00. 02000000:00000001:1.0:1775768161.971150:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768161.971151:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768161.971151:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fca80. 00000100:00000001:1.0:1775768161.971152:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768161.971153:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768161.971169:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768161.971173:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768161.971174:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768161.971176:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768161.971177:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768162.033233:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768162.033239:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768162.033241:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768162.033245:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768162.033249:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768162.033251:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768162.033253:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768162.033255:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768162.033259:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768162.033261:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768162.033264:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768162.033265:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768162.033269:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768162.033270:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768162.033273:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768162.033274:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768162.033325:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768162.033344:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768162.033347:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768162.097265:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768162.097271:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768162.097274:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768162.097278:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768162.097282:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768162.097286:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768162.097289:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768162.097291:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768162.097294:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768162.993304:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768162.993306:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768162.993308:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768162.993314:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768162.993315:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768162.993315:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768162.993331:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768162.993332:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768162.993333:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768162.993346:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:0.0:1775768162.993347:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775768162.993348:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768162.993362:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768162.993362:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768162.993367:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768162.993374:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768162.993375:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768162.993376:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768162.993380:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768162.993386:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768162.993390:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768162.993391:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768162.993394:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768162.993397:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768162.993398:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768162.993402:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768162.993406:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768163.057283:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768163.057293:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768163.057301:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768163.057302:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768163.057307:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768163.057309:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768163.057314:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768163.057327:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768163.057332:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768163.057333:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768163.057398:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768163.057404:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768163.057408:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768163.121269:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768163.121277:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768163.121279:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768163.121285:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768163.121289:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768163.121300:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768163.121303:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768163.121306:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768163.121309:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768163.313272:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4300088125, at 4300088128 00000100:00000010:1.0:1775768163.313286:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cba2a0. 00000100:00000040:0.0:1775768163.313346:0:15036:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4300088125, at 4300088128 00000100:00000010:0.0:1775768163.313361:0:15036:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cba0c0. 00000100:00000001:0.0:1775768164.017268:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768164.017268:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768164.017275:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768164.017276:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768164.017278:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768164.017279:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768164.017281:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:2.0:1775768164.017283:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:0.0:1775768164.017285:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768164.017287:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768164.017288:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768164.017291:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768164.017291:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768164.017292:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768164.017296:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768164.017298:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00100000:1.0:1775768164.017299:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768164.017301:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768164.017305:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768164.017305:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768164.017322:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768164.017326:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768164.017329:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768164.017336:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768164.017339:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768164.017342:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768164.017345:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768164.081254:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768164.081260:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768164.081267:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768164.081268:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768164.081273:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768164.081274:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768164.081277:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768164.081279:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768164.081282:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768164.081283:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768164.081336:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768164.081339:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768164.081342:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768164.145272:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768164.145280:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768164.145282:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768164.145288:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768164.145292:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768164.145297:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768164.145299:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768164.145302:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768164.145327:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768165.041246:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768165.041250:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768165.041254:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768165.041254:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768165.041257:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768165.041257:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768165.041263:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:2.0:1775768165.041263:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768165.041268:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768165.041268:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768165.041273:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768165.041273:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768165.041276:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768165.041278:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768165.041282:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768165.041282:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768165.041282:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768165.041284:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768165.041288:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768165.041289:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768165.041292:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768165.041309:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768165.041313:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768165.041316:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768165.041318:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768165.041321:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768165.041323:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768165.105252:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768165.105257:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768165.105261:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768165.105263:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768165.105266:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768165.105267:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768165.105270:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768165.105271:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768165.105273:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768165.105273:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768165.105338:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768165.105340:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768165.105343:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768165.169323:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768165.169330:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768165.169333:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768165.169344:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768165.169349:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768165.169353:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768165.169355:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768165.169357:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768165.169360:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768166.065258:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768166.065263:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768166.065265:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768166.065269:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768166.065272:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768166.065275:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768166.065276:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768166.065278:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768166.065280:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768166.065358:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768166.065361:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768166.065362:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768166.065365:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768166.065367:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768166.065368:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768166.065370:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768166.065372:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768166.065374:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768166.065381:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768166.065384:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768166.065385:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768166.065388:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768166.065390:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768166.065391:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768166.065393:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768166.065394:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768166.065396:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768166.129241:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768166.129245:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768166.129248:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768166.129249:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768166.129251:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768166.129252:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768166.129255:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768166.129256:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768166.129258:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768166.129258:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768166.129308:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768166.129312:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768166.129315:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768166.193272:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768166.193278:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768166.193280:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768166.193284:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768166.193308:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768166.193312:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768166.193314:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768166.193316:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768166.193318:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768167.089245:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768167.089245:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768167.089250:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768167.089252:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768167.089252:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768167.089255:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768167.089256:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775768167.089257:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768167.089259:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:2.0:1775768167.089261:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775768167.089262:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1775768167.089264:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775768167.089265:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768167.089266:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768167.089267:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768167.089267:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775768167.089268:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768167.089271:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768167.089272:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775768167.089272:0:36:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775768167.089273:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768167.089274:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768167.089289:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768167.089289:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768167.089289:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775768167.089290:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768167.089295:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768167.089295:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775768167.089297:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5419 00000100:00000001:1.0:1775768167.089303:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768167.089303:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768167.089307:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1775768167.089308:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768167.089312:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775768167.089316:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775768167.089318:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775768167.089321:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768167.089323:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775768167.089325:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768167.089327:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775768167.089329:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775768167.089331:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:3.0:1775768167.089332:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775768167.089334:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:3.0:1775768167.089336:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:3.0:1775768167.089337:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:3.0:1775768167.089339:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768167.089340:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775768167.089341:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768167.089342:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775768167.089344:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:3.0:1775768167.089346:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775768167.089348:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768167.089350:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775768167.089352:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775768167.089354:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768167.089355:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775768167.089357:0:36:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775768167.089363:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8380. 00000020:00000040:3.0:1775768167.089365:0:36:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:3.0:1775768167.089368:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775768167.089369:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768167.089371:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775768167.089373:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775768167.089376:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775768167.089383:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b3400. 00000100:00000001:3.0:1775768167.089387:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768167.089394:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8380 x1862022294282240/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775768167.089401:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775768167.089404:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768167.089407:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055f5621a] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8380 x1862022294282240/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768167.089420:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768167.089422:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768167.089424:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775768167.089425:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775768167.089426:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775768167.089428:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775768167.089429:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5419 00000400:00000200:3.0:1775768167.089432:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775768167.089433:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775768167.089435:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775768167.089437:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768167.089439:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775768167.089440:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768167.089441:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775768167.089443:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775768167.089443:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:3.0:1775768167.089444:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775768167.089445:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:3.0:1775768167.089446:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:3.0:1775768167.089448:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000001:1.0:1775768167.089450:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1775768167.089450:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768167.089451:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775768167.089454:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768167.089455:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768167.089456:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:1.0:1775768167.089458:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000040:3.0:1775768167.089460:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00080000:3.0:1775768167.089465:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000040:1.0:1775768167.089467:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a8380 x1862022294282240/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768167.089468:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768167.089471:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775768167.089478:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768167.089479:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768167.089481:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768167.089483:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768167.089485:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1775768167.089486:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:1.0:1775768167.089490:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00100000:3.0:1775768167.089492:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1775768167.089493:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1775768167.089497:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768167.089500:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294282240:192.168.202.156@tcp:250:kworker.0 00000100:00000001:3.0:1775768167.089501:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775768167.089505:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8700. 00000100:00000001:1.0:1775768167.089507:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000040:3.0:1775768167.089509:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:1.0:1775768167.089512:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775768167.089512:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775768167.089515:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768167.089516:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768167.089516:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775768167.089518:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775768167.089521:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1775768167.089521:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775768167.089526:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2000. 02000000:00000010:1.0:1775768167.089528:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543dc00. 00000100:00000001:3.0:1775768167.089529:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768167.089531:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768167.089535:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8700 x1862022294282368/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775768167.089541:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775768167.089544:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775768167.089546:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775768167.089548:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000100:00000040:3.0:1775768167.089548:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000faa25e3b] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8700 x1862022294282368/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1775768167.089556:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00000100:00000001:3.0:1775768167.089556:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768167.089558:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768167.089561:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775768167.089562:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775768167.089563:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:1.0:1775768167.089565:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294282240, portal 25 00000100:00000001:3.0:1775768167.089565:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775768167.089568:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5419 00000100:00000001:1.0:1775768167.089570:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768167.089573:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640853376 : -112966068698240 : ffff9942099a8380) 00000400:00000200:3.0:1775768167.089573:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775768167.089576:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775768167.089579:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775768167.089581:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768167.089583:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768167.089585:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a8380 x1862022294282240/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768192 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775768167.089586:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768167.089589:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775768167.089592:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775768167.089594:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:3.0:1775768167.089596:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775768167.089598:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:3.0:1775768167.089599:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:3.0:1775768167.089600:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:3.0:1775768167.089601:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768167.089602:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775768167.089603:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768167.089607:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775768167.089611:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:3.0:1775768167.089614:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775768167.089615:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1775768167.089616:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768167.089618:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775768167.089620:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775768167.089621:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768167.089623:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000040:1.0:1775768167.089625:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000008:00100000:3.0:1775768167.089626:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000400:00000010:1.0:1775768167.089630:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6880. 00000008:00000001:3.0:1775768167.089630:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768167.089634:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775768167.089636:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294282240, offset 0 00000100:00000010:3.0:1775768167.089638:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a9180. 00000020:00000040:3.0:1775768167.089641:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:3.0:1775768167.089644:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000200:1.0:1775768167.089645:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 02000000:00000001:3.0:1775768167.089647:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768167.089649:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775768167.089650:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775768167.089652:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775768167.089655:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b1000. 00000100:00000001:3.0:1775768167.089661:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768167.089662:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00100000:3.0:1775768167.089671:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a9180 x1862022294282496/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775768167.089672:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768167.089681:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1775768167.089683:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775768167.089686:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768167.089687:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768167.089693:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000100:00000040:3.0:1775768167.089693:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bdc2586] to pc [ptlrpcd_rcv+-1] req@ffff9942099a9180 x1862022294282496/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775768167.089700:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768167.089708:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225378:225378:256:4294967295] 192.168.202.156@tcp LPNI seq info [225374:225374:8:4294967295] 00000100:00000001:3.0:1775768167.089709:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768167.089713:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1775768167.089718:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00080000:3.0:1775768167.089719:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775768167.089721:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775768167.089722:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775768167.089725:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775768167.089729:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5419 00000400:00000200:1.0:1775768167.089733:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000400:00000200:3.0:1775768167.089736:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775768167.089741:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775768167.089744:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000800:00000200:1.0:1775768167.089745:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00000040:3.0:1775768167.089749:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000800:00000010:1.0:1775768167.089750:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541500. 00000100:00000001:3.0:1775768167.089751:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775768167.089754:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768167.089756:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775768167.089757:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775768167.089758:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000800:00000200:1.0:1775768167.089760:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000001:3.0:1775768167.089760:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775768167.089763:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:3.0:1775768167.089766:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000800:00000200:1.1:1775768167.089767:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000001:3.0:1775768167.089769:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:3.0:1775768167.089772:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768167.089773:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775768167.089774:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768167.089775:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775768167.089777:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:3.0:1775768167.089779:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775768167.089780:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768167.089780:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768167.089785:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1775768167.089787:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:3.0:1775768167.089787:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775768167.089791:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768167.089792:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4300093245 00000100:00000001:3.0:1775768167.089793:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775768167.089797:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000100:00000010:1.0:1775768167.089798:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba2a0. 00000008:00000001:3.0:1775768167.089799:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768167.089802:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775768167.089805:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ab480. 00000100:00000001:1.0:1775768167.089807:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000020:00000040:3.0:1775768167.089808:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:3.0:1775768167.089813:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775768167.089814:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768167.089815:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1775768167.089816:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4300093245 02000000:00000001:3.0:1775768167.089816:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775768167.089819:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775768167.089824:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba420. 02000000:00000010:3.0:1775768167.089824:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0c00. 00000100:00000001:3.0:1775768167.089826:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768167.089829:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00100000:3.0:1775768167.089831:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ab480 x1862022294282624/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775768167.089835:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768167.089841:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768167.089841:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775768167.089843:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768167.089847:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768167.089848:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000067a0ba74] to pc [ptlrpcd_rcv+-1] req@ffff9942099ab480 x1862022294282624/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768167.089852:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768167.089858:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768167.089860:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768167.089862:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775768167.089862:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775768167.089863:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:3.0:1775768167.089864:0:36:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768167.089867:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1775768167.089867:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775768167.089871:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5419 00000100:00000001:1.0:1775768167.089872:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:3.0:1775768167.089875:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768167.089876:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000400:00000200:3.0:1775768167.089878:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:1.0:1775768167.089880:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00080000:3.0:1775768167.089880:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775768167.089884:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768167.089885:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775768167.089888:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000040:1.0:1775768167.089889:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a8700 x1862022294282368/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768167.089889:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775768167.089892:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775768167.089893:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:3.0:1775768167.089896:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775768167.089897:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:3.0:1775768167.089900:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:3.0:1775768167.089903:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:3.0:1775768167.089908:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768167.089911:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775768167.089912:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000040:3.0:1775768167.089913:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768167.089915:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768167.089916:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775768167.089918:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 02000000:00000001:1.0:1775768167.089921:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:3.0:1775768167.089921:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 02000000:00000001:1.0:1775768167.089925:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775768167.089925:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768167.089926:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775768167.089929:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768167.089930:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768167.089930:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768167.089933:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775768167.089936:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000100:00100000:1.0:1775768167.089939:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294282368:192.168.202.156@tcp:38:kworker.0 00000008:00000001:3.0:1775768167.089939:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768167.089941:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775768167.089945:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8e00. 00000100:00000001:1.0:1775768167.089946:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000040:3.0:1775768167.089947:0:36:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:3.0:1775768167.089950:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768167.089951:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775768167.089951:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768167.089954:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775768167.089955:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768167.089955:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775768167.089957:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768167.089959:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775768167.089960:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0400. 00000100:00000001:3.0:1775768167.089963:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775768167.089964:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543e800. 00000100:00100000:3.0:1775768167.089967:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8e00 x1862022294282752/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1775768167.089968:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768167.089977:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1775768167.089978:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775768167.089980:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768167.089983:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000864a6866] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8e00 x1862022294282752/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1775768167.089984:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c2c0. 00000400:00000010:1.0:1775768167.089991:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6990. 00000100:00000001:3.0:1775768167.089992:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775768167.089997:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294282368, portal 10 00000100:00000040:3.0:1775768167.089997:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5429) 00000100:00000001:1.0:1775768167.090002:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768167.090003:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640854272 : -112966068697344 : ffff9942099a8700) 00000100:00000040:1.0:1775768167.090011:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a8700 x1862022294282368/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768192 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768167.090018:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768167.090021:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768167.090023:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf62a8. 00000100:00000200:1.0:1775768167.090026:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294282368, offset 0 00000400:00000200:1.0:1775768167.090030:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768167.090034:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768167.090037:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768167.090040:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768167.090043:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768167.090045:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768167.090047:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768167.090050:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225379:225379:255:4294967295] 192.168.202.156@tcp LPNI seq info [225375:225375:7:4294967295] 00000400:00000200:1.0:1775768167.090054:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768167.090060:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768167.090065:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768167.090069:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541600. 00000800:00000200:1.0:1775768167.090073:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768167.090077:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:1.0:1775768167.090079:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768167.090083:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:1.0:1775768167.090086:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768167.090088:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300091922 00000100:00000001:1.0:1775768167.090090:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300093245 00000100:00000001:1.0:1775768167.090092:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300091922 00000100:00000001:1.0:1775768167.090094:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300093245 00000100:00000001:1.0:1775768167.090096:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768167.090098:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768167.090100:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768167.090104:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a9180 x1862022294282496/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768167.090111:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768167.090112:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768167.090114:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768167.090116:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768167.090118:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768167.090121:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294282496:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768167.090124:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768167.090125:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768167.090127:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768167.090129:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768167.090132:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543fc00. 02000000:00000001:1.0:1775768167.090133:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768167.090137:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768167.090139:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c688. 00000400:00000010:1.0:1775768167.090142:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000100:00000200:1.0:1775768167.090145:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294282496, portal 10 00000100:00000001:1.0:1775768167.090147:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768167.090148:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640856960 : -112966068694656 : ffff9942099a9180) 00000100:00000040:1.0:1775768167.090153:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a9180 x1862022294282496/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768192 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768167.090181:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768167.090183:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768167.090186:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf66e8. 00000100:00000200:1.0:1775768167.090188:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294282496, offset 0 00000400:00000200:1.0:1775768167.090192:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768167.090196:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768167.090199:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768167.090202:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768167.090204:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768167.090206:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768167.090208:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768167.090211:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225380:225380:254:4294967295] 192.168.202.156@tcp LPNI seq info [225376:225376:6:4294967295] 00000400:00000200:1.0:1775768167.090215:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768167.090221:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768167.090232:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768167.090236:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541100. 00000800:00000200:1.0:1775768167.090239:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768167.090243:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:1.0:1775768167.090245:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768167.090249:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:1.0:1775768167.090252:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768167.090254:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300091915 00000100:00000001:1.0:1775768167.090256:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300093245 00000100:00000001:1.0:1775768167.090258:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300091915 00000100:00000001:1.0:1775768167.090260:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300093245 00000100:00000001:1.0:1775768167.090262:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768167.090265:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768167.090267:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768167.090271:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ab480 x1862022294282624/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768167.090290:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768167.090291:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768167.090294:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768167.090295:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768167.090298:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768167.090301:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ab480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294282624:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768167.090304:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768167.090306:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768167.090308:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768167.090310:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768167.090313:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543f800. 02000000:00000001:1.0:1775768167.090315:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768167.090318:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768167.090321:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c738. 00000400:00000010:1.0:1775768167.090324:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf65d8. 00000100:00000200:1.0:1775768167.090327:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294282624, portal 4 00000100:00000001:1.0:1775768167.090329:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768167.090330:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640865920 : -112966068685696 : ffff9942099ab480) 00000100:00000040:1.0:1775768167.090335:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ab480 x1862022294282624/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768192 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768167.090342:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768167.090344:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768167.090346:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6a18. 00000100:00000200:1.0:1775768167.090348:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294282624, offset 0 00000400:00000200:1.0:1775768167.090352:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768167.090356:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768167.090359:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768167.090362:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768167.090365:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768167.090367:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768167.090369:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768167.090372:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225381:225381:253:4294967295] 192.168.202.156@tcp LPNI seq info [225377:225377:5:4294967295] 00000400:00000200:1.0:1775768167.090376:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768167.090383:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768167.090388:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768167.090391:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541700. 00000800:00000200:1.0:1775768167.090395:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768167.090399:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:1.0:1775768167.090401:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768167.090405:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:1.0:1775768167.090407:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768167.090409:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300091938 00000100:00000001:1.0:1775768167.090411:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300093245 00000100:00000001:1.0:1775768167.090413:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300091938 00000100:00000001:1.0:1775768167.090415:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300093245 00000100:00000001:1.0:1775768167.090417:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768167.090419:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768167.090422:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768167.090424:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768167.090428:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768167.090429:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1775768167.090431:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768167.090432:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768167.090433:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768167.090437:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a8e00 x1862022294282752/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768167.090444:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768167.090446:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768167.090448:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768167.090449:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768167.090452:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768167.090455:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294282752:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768167.090458:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768167.090459:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768167.090461:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768167.090463:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768167.090466:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543c800. 02000000:00000001:1.0:1775768167.090468:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768167.090471:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768167.090474:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023cbb0. 00000400:00000010:1.0:1775768167.090477:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6cc0. 00000100:00000200:1.0:1775768167.090480:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294282752, portal 4 00000100:00000001:1.0:1775768167.090481:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768167.090483:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640856064 : -112966068695552 : ffff9942099a8e00) 00000100:00000040:1.0:1775768167.090489:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a8e00 x1862022294282752/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768192 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768167.090496:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768167.090498:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768167.090501:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf67f8. 00000100:00000200:1.0:1775768167.090503:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294282752, offset 0 00000400:00000200:1.0:1775768167.090507:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768167.090511:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768167.090515:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768167.090517:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768167.090520:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768167.090522:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768167.090524:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768167.090527:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225382:225382:252:4294967295] 192.168.202.156@tcp LPNI seq info [225378:225378:4:4294967295] 00000400:00000200:1.0:1775768167.090530:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768167.090536:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768167.090541:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768167.090546:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541200. 00000800:00000200:1.0:1775768167.090549:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768167.090552:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:1.0:1775768167.090555:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768167.090558:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:1.0:1775768167.090561:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768167.090563:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300091941 00000100:00000001:1.0:1775768167.090565:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300093245 00000100:00000001:1.0:1775768167.090567:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300091941 00000100:00000001:1.0:1775768167.090569:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300093245 00000100:00000001:1.0:1775768167.090571:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768167.090573:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768167.090575:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768167.090577:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768167.090580:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768167.090581:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1775768167.090583:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768167.090584:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768167.090586:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768167.090588:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768167.153255:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768167.153260:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768167.153266:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768167.153269:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768167.153271:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768167.153275:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768167.153286:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768167.153289:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768167.153291:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768167.153293:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768167.153297:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768167.153298:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768167.153300:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768167.153301:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768167.153303:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768167.153304:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768167.153306:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768167.153307:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768167.153311:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768167.153312:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768167.153313:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768167.153314:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768167.153316:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768167.153317:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768167.153318:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768167.153320:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768167.153322:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768167.153323:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768167.153324:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768167.153325:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768167.153326:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768167.153326:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768167.153327:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768167.153329:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768167.153376:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768167.153379:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768167.153381:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768167.217258:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768167.217265:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768167.217267:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768167.217272:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768167.217299:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768167.217303:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768167.217306:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768167.217308:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768167.217310:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768168.113236:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768168.113236:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768168.113242:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768168.113243:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768168.113244:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768168.113245:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768168.113249:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:0.0:1775768168.113250:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768168.113253:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768168.113255:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768168.113256:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768168.113257:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768168.113259:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768168.113261:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768168.113263:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768168.113266:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768168.113287:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768168.113290:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768168.113306:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768168.113310:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768168.113312:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768168.113315:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768168.113317:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768168.113319:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768168.113322:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768168.113323:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768168.113325:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768168.177239:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768168.177245:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768168.177250:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768168.177252:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768168.177255:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768168.177256:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768168.177259:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768168.177260:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768168.177263:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768168.177264:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768168.177324:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768168.177327:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768168.177329:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768168.241233:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768168.241237:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768168.241239:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768168.241242:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768168.241245:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768168.241248:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768168.241250:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768168.241252:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768168.241253:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768168.433245:0:15036:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4300093245, at 4300093248 00000100:00000040:1.0:1775768168.433245:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4300093245, at 4300093248 00000100:00000010:0.0:1775768168.433260:0:15036:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cba2a0. 00000100:00000010:1.0:1775768168.433263:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cba420. 00000100:00000001:1.0:1775768169.137219:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768169.137223:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768169.137224:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768169.137227:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768169.137228:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768169.137231:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768169.137231:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:0.0:1775768169.137236:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768169.137236:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768169.137240:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768169.137240:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768169.137244:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768169.137244:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768169.137248:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768169.137251:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768169.137253:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768169.137270:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768169.137273:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768169.137288:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768169.137292:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768169.137293:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768169.137296:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768169.137298:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768169.137300:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768169.137301:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768169.137302:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768169.137304:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768169.201272:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768169.201277:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768169.201283:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768169.201285:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768169.201289:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768169.201290:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768169.201293:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768169.201294:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768169.201297:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768169.201298:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768169.201348:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768169.201351:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768169.201356:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768169.265254:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768169.265280:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768169.265281:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768169.265286:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768169.265288:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768169.265291:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768169.265293:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768169.265295:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768169.265296:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768170.161304:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768170.161307:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768170.161310:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768170.161315:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768170.161316:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768170.161320:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768170.161321:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768170.161329:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:0.0:1775768170.161330:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768170.161334:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768170.161335:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768170.161341:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768170.161342:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768170.161346:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768170.161351:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768170.161353:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768170.161358:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768170.161362:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768170.161483:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768170.161487:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768170.161489:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768170.161492:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768170.161495:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768170.161497:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768170.161499:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768170.161500:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768170.161502:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:3.0:1775768170.161596:0:2389:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.156:988 00000400:00000100:3.0:1775768170.161642:0:2389:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.156@tcp at host 192.168.202.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:3.0:1775768170.161653:0:2389:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775768170.161657:0:2389:0:(router.c:1750:lnet_notify()) 192.168.202.56@tcp notifying 192.168.202.156@tcp: down 00000400:00000200:3.0:1775768170.161662:0:2389:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775768170.161666:0:2389:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:3.0:1775768170.161672:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000800:00000010:3.0:1775768170.161677:0:2389:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541500. 00000400:00000200:3.0:1775768170.161682:0:2389:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1775768170.161687:0:2389:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1775768170.161694:0:2389:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6880 00000400:00000010:3.0:1775768170.161697:0:2389:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6880. 00000100:00000001:3.0:1775768170.161712:0:2389:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775768170.161723:0:2389:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9942099a8380 x1862022294282240/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768192 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768170.161750:0:2389:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775768170.161754:0:2389:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a8380 x1862022294282240/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768192 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768170.161762:0:2389:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768170.161764:0:2389:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1775768170.161778:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000800:00000010:3.0:1775768170.161781:0:2389:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541600. 00000400:00000200:3.0:1775768170.161784:0:2389:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000100:00000001:1.0:1775768170.161788:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:3.0:1775768170.161790:0:2389:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000100:00000001:1.0:1775768170.161793:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000400:00000200:3.0:1775768170.161794:0:2389:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf62a8 00000400:00000010:3.0:1775768170.161797:0:2389:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf62a8. 00000100:00000001:1.0:1775768170.161800:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1775768170.161800:0:2389:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775768170.161806:0:2389:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9942099a8700 x1862022294282368/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768192 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768170.161814:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768167/real 1775768170] req@ffff9942099a8380 x1862022294282240/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768170.161819:0:2389:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775768170.161825:0:2389:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a8700 x1862022294282368/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768192 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768170.161835:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1775768170.161835:0:2389:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1775768170.161839:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1775768170.161839:0:2389:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000010:1.0:1775768170.161844:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000800:00000100:3.0:1775768170.161845:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000800:00000010:3.0:1775768170.161850:0:2389:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541100. 00000400:00000200:1.0:1775768170.161852:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000200:3.0:1775768170.161855:0:2389:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000010:1.0:1775768170.161856:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000400:00000200:3.0:1775768170.161861:0:2389:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000100:00000001:1.0:1775768170.161863:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000200:3.0:1775768170.161864:0:2389:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf66e8 00000400:00000010:3.0:1775768170.161867:0:2389:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf66e8. 00000100:00000200:1.0:1775768170.161870:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a8380 x1862022294282240/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768170.161872:0:2389:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775768170.161880:0:2389:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9942099a9180 x1862022294282496/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768192 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768170.161890:0:2389:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000200:1.0:1775768170.161892:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a8380 x1862022294282240/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775768170.161896:0:2389:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a9180 x1862022294282496/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768192 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768170.161915:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775768170.161915:0:2389:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768170.161918:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768170.161918:0:2389:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768170.161924:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000800:00000100:3.0:1775768170.161927:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000100:00000001:1.0:1775768170.161928:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:3.0:1775768170.161931:0:2389:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541700. 00000400:00000200:3.0:1775768170.161935:0:2389:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000100:00100000:1.0:1775768170.161936:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a8380 x1862022294282240/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1775768170.161942:0:2389:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000100:00000001:1.0:1775768170.161947:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768170.161954:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a8380 x1862022294282240/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768192 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1775768170.161963:0:2389:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6a18 00000100:00000001:1.0:1775768170.161967:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000400:00000010:3.0:1775768170.161967:0:2389:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6a18. 00000100:00000001:1.0:1775768170.161973:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768170.161973:0:2389:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1775768170.161976:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768170.161978:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:3.0:1775768170.161981:0:2389:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9942099ab480 x1862022294282624/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768192 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768170.161982:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768170.161986:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768170.161993:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1775768170.161996:0:2389:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775768170.162001:0:2389:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ab480 x1862022294282624/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768192 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768170.162003:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768170.162010:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1775768170.162012:0:2389:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768170.162015:0:2389:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:1.0:1775768170.162017:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a8380 x1862022294282240/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768192 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:3.0:1775768170.162026:0:2389:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000800:00000010:3.0:1775768170.162030:0:2389:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541200. 00000100:00100000:1.0:1775768170.162035:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294282240:192.168.202.156@tcp:250:kworker.0 00000400:00000200:3.0:1775768170.162035:0:2389:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1775768170.162042:0:2389:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000100:00000001:1.0:1775768170.162043:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775768170.162046:0:2389:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf67f8 00000100:00000001:1.0:1775768170.162047:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000400:00000010:3.0:1775768170.162050:0:2389:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf67f8. 00000100:00000040:1.0:1775768170.162053:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a8380 x1862022294282240/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768192 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768170.162054:0:2389:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775768170.162062:0:2389:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9942099a8e00 x1862022294282752/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768192 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768170.162064:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768170.162068:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1775768170.162073:0:2389:0:(client.c:2843:__ptlrpc_req_put()) Process entered 02000000:00000010:1.0:1775768170.162075:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543dc00. 00000100:00000040:3.0:1775768170.162076:0:2389:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a8e00 x1862022294282752/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768192 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1775768170.162082:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1775768170.162086:0:2389:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768170.162088:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:3.0:1775768170.162089:0:2389:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000040:1.0:1775768170.162094:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:1.0:1775768170.162100:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768170.162105:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b3400. 02000000:00000001:1.0:1775768170.162111:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768170.162113:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768170.162116:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a8380. 00000100:00000001:1.0:1775768170.162121:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768170.162122:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768170.162126:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768170.162128:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768170.162129:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768170.162130:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768170.162140:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768167/real 1775768170] req@ffff9942099a8e00 x1862022294282752/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768170.162148:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768170.162150:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768170.162152:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023cbb0. 00000400:00000200:1.0:1775768170.162178:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6cc0 00000400:00000010:1.0:1775768170.162180:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6cc0. 00000100:00000001:1.0:1775768170.162183:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768170.162187:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a8e00 x1862022294282752/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768170.162196:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a8e00 x1862022294282752/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768170.162203:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768170.162204:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768170.162207:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768170.162208:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768170.162212:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a8e00 x1862022294282752/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768170.162217:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768170.162220:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a8e00 x1862022294282752/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768192 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768170.162226:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768170.162229:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768170.162231:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768170.162232:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768170.162234:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768170.162238:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768170.162241:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768170.162245:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768170.162258:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768170.162262:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a8e00 x1862022294282752/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768192 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768170.162269:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294282752:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768170.162272:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768170.162277:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768167/real 1775768170] req@ffff9942099a8700 x1862022294282368/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768170.162284:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768170.162285:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768170.162287:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c2c0. 00000400:00000200:1.0:1775768170.162296:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6990 00000400:00000010:1.0:1775768170.162297:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6990. 00000100:00000001:1.0:1775768170.162310:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768170.162317:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a8700 x1862022294282368/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768170.162324:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a8700 x1862022294282368/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768170.162330:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768170.162331:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768170.162333:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768170.162334:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768170.162338:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a8700 x1862022294282368/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768170.162345:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768170.162349:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a8700 x1862022294282368/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768192 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768170.162354:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768170.162356:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768170.162358:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768170.162360:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768170.162362:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768170.162365:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768170.162368:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768170.162372:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768170.162375:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768170.162379:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a8700 x1862022294282368/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768192 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768170.162386:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294282368:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768170.162391:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768170.162395:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768167/real 1775768170] req@ffff9942099a9180 x1862022294282496/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768170.162401:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768170.162402:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768170.162404:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c688. 00000400:00000200:1.0:1775768170.162406:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000400:00000010:1.0:1775768170.162408:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000100:00000001:1.0:1775768170.162410:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768170.162412:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a9180 x1862022294282496/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768170.162424:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a9180 x1862022294282496/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768170.162430:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768170.162431:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768170.162433:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768170.162434:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768170.162437:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a9180 x1862022294282496/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768170.162443:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768170.162446:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a9180 x1862022294282496/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768192 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768170.162452:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768170.162454:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768170.162456:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768170.162457:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768170.162458:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768170.162461:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768170.162464:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768170.162467:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768170.162470:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768170.162473:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a9180 x1862022294282496/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768192 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768170.162480:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294282496:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768170.162484:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768170.162487:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768167/real 1775768170] req@ffff9942099ab480 x1862022294282624/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768170.162500:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768170.162502:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768170.162503:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c738. 00000400:00000200:1.0:1775768170.162506:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf65d8 00000400:00000010:1.0:1775768170.162507:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf65d8. 00000100:00000001:1.0:1775768170.162509:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768170.162512:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ab480 x1862022294282624/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768170.162519:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ab480 x1862022294282624/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768170.162530:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768170.162531:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768170.162533:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768170.162534:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768170.162538:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ab480 x1862022294282624/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768170.162543:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768170.162546:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ab480 x1862022294282624/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768192 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768170.162553:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768170.162555:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768170.162556:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768170.162557:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768170.162559:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768170.162563:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768170.162566:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768170.162570:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768170.162572:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768170.162576:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ab480 x1862022294282624/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768192 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768170.162584:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ab480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294282624:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768170.162587:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768170.162589:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768170.162592:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a8e00 x1862022294282752/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768192 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768170.162597:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768170.162599:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768170.162600:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543c800. 02000000:00000001:1.0:1775768170.162604:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768170.162605:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768170.162606:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:1.0:1775768170.162608:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768170.162609:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b0400. 02000000:00000001:1.0:1775768170.162613:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768170.162614:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768170.162616:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a8e00. 00000100:00000001:1.0:1775768170.162618:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768170.162619:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768170.162621:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768170.162624:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a8700 x1862022294282368/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768192 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768170.162630:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768170.162631:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768170.162633:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543e800. 02000000:00000001:1.0:1775768170.162636:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768170.162637:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768170.162638:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:1.0:1775768170.162641:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768170.162642:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b2000. 02000000:00000001:1.0:1775768170.162645:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768170.162647:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768170.162648:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a8700. 00000100:00000001:1.0:1775768170.162651:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768170.162653:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768170.162654:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768170.162657:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a9180 x1862022294282496/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768192 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768170.162663:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768170.162665:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768170.162666:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543fc00. 02000000:00000001:1.0:1775768170.162668:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768170.162669:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768170.162670:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:1.0:1775768170.162672:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768170.162673:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b1000. 02000000:00000001:1.0:1775768170.162676:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768170.162677:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768170.162678:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a9180. 00000100:00000001:1.0:1775768170.162680:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768170.162682:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768170.162683:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768170.162686:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ab480 x1862022294282624/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768192 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768170.162691:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768170.162693:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768170.162694:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543f800. 02000000:00000001:1.0:1775768170.162696:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768170.162698:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768170.162699:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:1.0:1775768170.162701:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768170.162702:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b0c00. 02000000:00000001:1.0:1775768170.162704:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768170.162705:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768170.162707:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ab480. 00000100:00000001:1.0:1775768170.162709:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768170.162711:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768170.162712:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768170.162721:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768170.162723:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768170.162725:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768170.162726:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768170.225265:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768170.225273:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768170.225281:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768170.225283:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768170.225287:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768170.225289:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768170.225292:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768170.225293:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768170.225298:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768170.225299:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768170.225367:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768170.225372:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768170.225376:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768170.289232:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768170.289239:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768170.289241:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768170.289257:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768170.289261:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768170.289265:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768170.289267:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768170.289269:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768170.289272:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768171.185256:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768171.185264:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768171.185266:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768171.185269:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:0.0:1775768171.185271:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768171.185272:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768171.185276:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768171.185280:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1775768171.185281:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775768171.185286:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768171.185286:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768171.185286:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768171.185293:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768171.185295:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768171.185295:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768171.185302:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768171.185302:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768171.185309:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768171.185310:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768171.185314:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768171.185315:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768171.185321:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768171.185323:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768171.185324:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768171.185330:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768171.185332:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768171.185336:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768171.249272:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768171.249277:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768171.249283:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768171.249284:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768171.249287:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768171.249288:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768171.249291:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768171.249292:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768171.249295:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768171.249297:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768171.249353:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768171.249357:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768171.249362:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768171.313266:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768171.313273:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768171.313275:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768171.313281:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768171.313284:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768171.313288:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768171.313290:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768171.313293:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768171.313296:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768172.209272:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768172.209272:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768172.209272:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768172.209279:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768172.209279:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768172.209283:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768172.209283:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768172.209285:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:1.0:1775768172.209289:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775768172.209289:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768172.209292:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:0.0:1775768172.209293:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768172.209294:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768172.209296:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768172.209296:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768172.209298:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768172.209300:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768172.209300:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768172.209301:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768172.209304:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768172.209308:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768172.209309:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768172.209311:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768172.209314:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768172.209314:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768172.209316:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768172.209318:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768172.209334:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768172.209339:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775768172.209342:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775768172.209346:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768172.209349:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775768172.209356:0:13471:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5424 00000400:00000200:1.0:1775768172.209364:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768172.209370:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775768172.209373:0:13471:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775768172.209376:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768172.209379:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768172.209381:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768172.209383:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775768172.209386:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775768172.209388:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:1.0:1775768172.209390:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775768172.209392:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:1.0:1775768172.209394:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:1.0:1775768172.209396:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:1.0:1775768172.209399:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768172.209401:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768172.209402:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768172.209404:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775768172.209407:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:1.0:1775768172.209409:0:13471:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775768172.209412:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768172.209414:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775768172.209417:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768172.209420:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768172.209422:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775768172.209424:0:13471:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775768172.209431:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207889f80. 00000020:00000040:1.0:1775768172.209435:0:13471:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:1.0:1775768172.209439:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768172.209440:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768172.209444:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775768172.209446:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775768172.209450:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775768172.209456:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543f800. 00000100:00000001:1.0:1775768172.209462:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768172.209471:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207889f80 x1862022294282880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775768172.209486:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775768172.209491:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768172.209495:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ed2274f4] to pc [ptlrpcd_rcv+-1] req@ffff994207889f80 x1862022294282880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768172.209526:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768172.209529:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768172.209532:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775768172.209533:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775768172.209535:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768172.209537:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775768172.209540:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5424 00000400:00000200:1.0:1775768172.209543:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768172.209546:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775768172.209548:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775768172.209551:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768172.209553:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768172.209555:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768172.209557:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775768172.209559:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775768172.209560:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:1.0:1775768172.209562:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775768172.209564:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:1.0:1775768172.209566:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:1.0:1775768172.209567:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:1.0:1775768172.209570:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768172.209572:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768172.209574:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768172.209576:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775768172.209579:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00080000:1.0:1775768172.209581:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775768172.209583:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768172.209585:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775768172.209590:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768172.209592:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768172.209594:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775768172.209598:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775768172.209600:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768172.209602:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775768172.209605:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff99420788b100. 00000020:00000040:1.0:1775768172.209608:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:1.0:1775768172.209610:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768172.209611:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768172.209613:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775768172.209614:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775768172.209615:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775768172.209619:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543fc00. 00000100:00000001:1.0:1775768172.209622:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768172.209627:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff99420788b100 x1862022294283008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775768172.209633:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775768172.209635:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768172.209638:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e26cf977] to pc [ptlrpcd_rcv+-1] req@ffff99420788b100 x1862022294283008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768172.209645:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768172.209647:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768172.209650:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775768172.209651:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775768172.209652:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768172.209654:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775768172.209657:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5424 00000400:00000200:1.0:1775768172.209660:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768172.209663:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775768172.209665:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775768172.209668:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768172.209669:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768172.209671:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768172.209673:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775768172.209674:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775768172.209675:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:1.0:1775768172.209677:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775768172.209679:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:1.0:1775768172.209681:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:1.0:1775768172.209683:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:1.0:1775768172.209685:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768172.209686:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768172.209688:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768172.209689:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775768172.209691:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:1.0:1775768172.209693:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775768172.209695:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768172.209696:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775768172.209700:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768172.209701:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768172.209703:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775768172.209705:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775768172.209706:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768172.209708:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775768172.209710:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207888700. 00000020:00000040:1.0:1775768172.209712:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:1.0:1775768172.209714:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768172.209715:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768172.209717:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775768172.209718:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775768172.209720:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775768172.209723:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543e800. 00000100:00000001:1.0:1775768172.209726:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768172.209730:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207888700 x1862022294283136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775768172.209736:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775768172.209738:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768172.209741:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009f2ba301] to pc [ptlrpcd_rcv+-1] req@ffff994207888700 x1862022294283136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768172.209748:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768172.209750:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768172.209753:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775768172.209755:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775768172.209756:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768172.209758:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775768172.209760:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5424 00000400:00000200:1.0:1775768172.209764:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768172.209767:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775768172.209769:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775768172.209771:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768172.209773:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768172.209775:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768172.209777:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775768172.209779:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775768172.209780:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:1.0:1775768172.209781:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775768172.209783:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:1.0:1775768172.209784:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:1.0:1775768172.209787:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:1.0:1775768172.209789:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768172.209791:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768172.209792:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768172.209794:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775768172.209795:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:1.0:1775768172.209797:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775768172.209799:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768172.209801:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775768172.209803:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768172.209805:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768172.209807:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775768172.209809:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:1.0:1775768172.209811:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768172.209812:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775768172.209815:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207889880. 00000020:00000040:1.0:1775768172.209817:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:1.0:1775768172.209819:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768172.209820:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768172.209822:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775768172.209824:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775768172.209825:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775768172.209827:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543c800. 00000100:00000001:1.0:1775768172.209830:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768172.209833:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207889880 x1862022294283264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775768172.209838:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775768172.209840:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768172.209843:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f11856b8] to pc [ptlrpcd_rcv+-1] req@ffff994207889880 x1862022294283264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768172.209856:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768172.209858:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768172.209861:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775768172.209862:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775768172.209863:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768172.209865:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775768172.209867:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5424 00000400:00000200:1.0:1775768172.209870:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768172.209872:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775768172.209874:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775768172.209876:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768172.209878:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768172.209880:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768172.209882:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775768172.209883:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775768172.209884:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:1.0:1775768172.209885:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775768172.209887:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:1.0:1775768172.209888:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:1.0:1775768172.209890:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:1.0:1775768172.209892:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768172.209894:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768172.209895:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768172.209896:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775768172.209898:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:1.0:1775768172.209900:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775768172.209902:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768172.209903:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775768172.209906:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768172.209908:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768172.209910:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775768172.209912:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:1.0:1775768172.209913:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768172.209915:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775768172.209917:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207888e00. 00000020:00000040:1.0:1775768172.209919:0:13471:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:1.0:1775768172.209921:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768172.209922:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768172.209923:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775768172.209924:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775768172.209926:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775768172.209929:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543dc00. 00000100:00000001:1.0:1775768172.209931:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768172.209936:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207888e00 x1862022294283392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775768172.209941:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775768172.209943:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768172.209946:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001812f966] to pc [ptlrpcd_rcv+-1] req@ffff994207888e00 x1862022294283392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768172.209952:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768172.209957:0:13471:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5434) 00000100:00000001:2.0:1775768172.210040:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768172.210044:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768172.210046:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768172.210052:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207889f80 x1862022294282880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768172.210060:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768172.210061:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768172.210065:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768172.210068:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768172.210070:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768172.210074:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207889f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294282880:192.168.202.156@tcp:250:kworker.0 00000100:00000001:2.0:1775768172.210079:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768172.210084:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768172.210086:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768172.210089:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768172.210094:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1e400. 02000000:00000001:2.0:1775768172.210096:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768172.210102:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768172.210106:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768172.210113:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768172.210123:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294282880, portal 25 00000100:00000001:2.0:1775768172.210126:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768172.210127:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606126464 : -112966103425152 : ffff994207889f80) 00000100:00000040:2.0:1775768172.210132:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207889f80 x1862022294282880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768197 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768172.210139:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768172.210144:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768172.210147:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768172.210151:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294282880, offset 0 00000400:00000200:2.0:1775768172.210184:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768172.210194:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768172.210201:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768172.210207:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768172.210212:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768172.210216:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768172.210219:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768172.210223:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225383:225383:256:4294967295] 192.168.202.156@tcp LPNI seq info [225379:225379:8:4294967295] 00000400:00000200:2.0:1775768172.210239:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768172.210250:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768172.210256:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768172.210263:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768172.210271:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768172.210275:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775768172.210281:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768172.210284:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1775768172.210288:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768172.210291:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768172.210298:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768172.210303:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768172.210310:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768172.210313:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768172.210318:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768172.210322:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207889f80 x1862022294282880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768197 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768172.210332:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768172.210336:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207889f80 x1862022294282880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768197 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768172.210343:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768172.210344:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768172.210350:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768172.210354:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768172.210357:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4300098366 00000100:00000010:2.0:1775768172.210363:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d6300. 00000100:00000001:2.0:1775768172.210372:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1775768172.210378:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4300098366 00000100:00000010:2.0:1775768172.210380:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d6480. 00000100:00000001:2.0:1775768172.210384:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1775768172.210387:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768172.210390:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768172.210393:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768172.210397:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768172/real 1775768172] req@ffff994207889f80 x1862022294282880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768172.210405:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768172.210407:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768172.210409:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768172.210411:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768172.210413:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768172.210415:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768172.210419:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207889f80 x1862022294282880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768172.210426:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207889f80 x1862022294282880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768172.210432:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768172.210433:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768172.210435:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768172.210437:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768172.210441:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207889f80 x1862022294282880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768172.210447:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768172.210452:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207889f80 x1862022294282880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768197 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768172.210458:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768172.210460:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768172.210462:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768172.210463:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768172.210465:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768172.210467:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768172.210470:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768172.210474:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768172.210476:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768172.210480:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207889f80 x1862022294282880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768197 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768172.210487:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207889f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294282880:192.168.202.156@tcp:250:kworker.0 00000100:00000001:2.0:1775768172.210492:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768172.210494:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff99420788b100 x1862022294283008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768172.210500:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768172.210501:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768172.210503:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768172.210504:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768172.210506:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768172.210508:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff99420788b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294283008:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768172.210511:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768172.210513:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768172.210514:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768172.210516:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768172.210519:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1d000. 02000000:00000001:2.0:1775768172.210521:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768172.210524:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768172.210526:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768172.210529:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768172.210532:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294283008, portal 10 00000100:00000001:2.0:1775768172.210533:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768172.210534:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606130944 : -112966103420672 : ffff99420788b100) 00000100:00000040:2.0:1775768172.210540:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff99420788b100 x1862022294283008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768197 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768172.210546:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768172.210548:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768172.210550:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768172.210552:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294283008, offset 0 00000400:00000200:2.0:1775768172.210556:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768172.210561:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768172.210564:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768172.210567:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768172.210569:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768172.210572:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768172.210574:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768172.210577:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225384:225384:256:4294967295] 192.168.202.156@tcp LPNI seq info [225380:225380:8:4294967295] 00000400:00000200:2.0:1775768172.210581:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768172.210587:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768172.210592:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768172.210596:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768172.210599:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768172.210602:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775768172.210606:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768172.210609:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1775768172.210612:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768172.210614:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768172.210617:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768172.210620:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768172.210623:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768172.210626:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768172.210629:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768172.210633:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff99420788b100 x1862022294283008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768197 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768172.210640:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768172.210642:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff99420788b100 x1862022294283008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768197 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768172.210648:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768172.210650:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768172.210653:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768172.210656:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:2.0:1775768172.210660:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768172.210661:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300097043 00000100:00000001:2.0:1775768172.210664:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300098366 00000100:00000001:2.0:1775768172.210666:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300097043 00000100:00000001:2.0:1775768172.210668:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300098366 00000100:00000001:2.0:1775768172.210671:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768172.210672:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768172.210674:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768172.210677:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768172/real 1775768172] req@ffff99420788b100 x1862022294283008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768172.210684:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768172.210686:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768172.210687:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768172.210689:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768172.210691:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768172.210693:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768172.210695:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff99420788b100 x1862022294283008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768172.210703:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff99420788b100 x1862022294283008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768172.210713:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768172.210714:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768172.210716:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768172.210717:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768172.210720:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff99420788b100 x1862022294283008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768172.210725:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768172.210728:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff99420788b100 x1862022294283008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768197 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768172.210734:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768172.210735:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768172.210737:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768172.210738:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768172.210740:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768172.210743:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768172.210745:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768172.210748:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768172.210750:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768172.210752:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff99420788b100 x1862022294283008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768197 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768172.210758:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff99420788b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294283008:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768172.210762:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768172.210765:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207888700 x1862022294283136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768172.210769:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768172.210771:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768172.210773:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768172.210774:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768172.210776:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768172.210778:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207888700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294283136:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768172.210781:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768172.210782:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768172.210784:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768172.210785:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768172.210788:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f400. 02000000:00000001:2.0:1775768172.210789:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768172.210792:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768172.210794:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768172.210796:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768172.210799:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294283136, portal 10 00000100:00000001:2.0:1775768172.210802:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768172.210803:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606120192 : -112966103431424 : ffff994207888700) 00000100:00000040:2.0:1775768172.210807:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207888700 x1862022294283136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768197 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768172.210812:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768172.210814:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768172.210816:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768172.210818:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294283136, offset 0 00000400:00000200:2.0:1775768172.210821:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768172.210825:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768172.210828:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768172.210830:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768172.210832:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768172.210834:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768172.210836:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768172.210839:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225385:225385:256:4294967295] 192.168.202.156@tcp LPNI seq info [225381:225381:8:4294967295] 00000400:00000200:2.0:1775768172.210842:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768172.210847:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768172.210852:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768172.210855:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768172.210859:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768172.210861:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775768172.210865:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768172.210867:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1775768172.210871:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768172.210872:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768172.210875:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768172.210878:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768172.210880:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768172.210882:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768172.210884:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768172.210887:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207888700 x1862022294283136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768197 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768172.210894:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768172.210897:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207888700 x1862022294283136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768197 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768172.210902:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768172.210904:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768172.210906:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768172.210910:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:2.0:1775768172.210913:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768172.210915:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300097035 00000100:00000001:2.0:1775768172.210917:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300098366 00000100:00000001:2.0:1775768172.210918:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300097035 00000100:00000001:2.0:1775768172.210920:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300098366 00000100:00000001:2.0:1775768172.210922:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768172.210924:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768172.210926:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768172.210930:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768172/real 1775768172] req@ffff994207888700 x1862022294283136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768172.210936:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768172.210937:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768172.210939:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768172.210940:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768172.210942:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768172.210944:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768172.210946:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207888700 x1862022294283136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768172.210954:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207888700 x1862022294283136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768172.210960:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768172.210961:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768172.210963:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768172.210964:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768172.210967:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207888700 x1862022294283136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768172.210973:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768172.210975:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207888700 x1862022294283136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768197 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768172.210981:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768172.210983:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768172.210984:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768172.210985:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768172.210986:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768172.210990:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768172.210992:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768172.210995:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768172.210997:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768172.211000:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207888700 x1862022294283136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768197 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768172.211006:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207888700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294283136:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768172.211010:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768172.211013:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207889880 x1862022294283264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768172.211018:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768172.211020:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768172.211021:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768172.211023:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768172.211025:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768172.211027:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207889880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294283264:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768172.211029:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768172.211031:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768172.211032:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768172.211034:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768172.211037:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1fc00. 02000000:00000001:2.0:1775768172.211039:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768172.211042:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768172.211045:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768172.211047:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768172.211050:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294283264, portal 4 00000100:00000001:2.0:1775768172.211052:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768172.211053:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606124672 : -112966103426944 : ffff994207889880) 00000100:00000040:2.0:1775768172.211058:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207889880 x1862022294283264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768197 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768172.211065:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768172.211067:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768172.211069:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768172.211071:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294283264, offset 0 00000400:00000200:2.0:1775768172.211075:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768172.211079:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768172.211081:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768172.211083:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768172.211085:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768172.211087:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768172.211089:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768172.211092:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225386:225386:256:4294967295] 192.168.202.156@tcp LPNI seq info [225382:225382:8:4294967295] 00000400:00000200:2.0:1775768172.211096:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768172.211101:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768172.211105:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768172.211109:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768172.211112:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768172.211115:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775768172.211118:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768172.211121:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1775768172.211124:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768172.211126:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768172.211129:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768172.211132:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768172.211135:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768172.211136:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768172.211138:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768172.211141:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207889880 x1862022294283264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768197 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768172.211146:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768172.211148:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207889880 x1862022294283264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768197 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768172.211151:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768172.211152:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768172.211174:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768172.211176:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:2.0:1775768172.211178:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768172.211180:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300097059 00000100:00000001:2.0:1775768172.211182:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300098366 00000100:00000001:2.0:1775768172.211183:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300097059 00000100:00000001:2.0:1775768172.211184:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300098366 00000100:00000001:2.0:1775768172.211185:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768172.211186:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768172.211188:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768172.211190:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768172/real 1775768172] req@ffff994207889880 x1862022294283264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768172.211194:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768172.211195:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768172.211196:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768172.211197:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768172.211198:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768172.211199:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768172.211201:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207889880 x1862022294283264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768172.211206:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207889880 x1862022294283264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768172.211210:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768172.211210:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768172.211211:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768172.211212:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768172.211214:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207889880 x1862022294283264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768172.211217:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768172.211219:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207889880 x1862022294283264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768197 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768172.211223:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768172.211224:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768172.211225:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768172.211226:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768172.211240:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768172.211244:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768172.211247:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768172.211250:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768172.211252:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768172.211255:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207889880 x1862022294283264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768197 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768172.211262:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207889880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294283264:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768172.211265:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768172.211268:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207888e00 x1862022294283392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768172.211273:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768172.211274:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768172.211276:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768172.211278:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768172.211280:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768172.211282:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207888e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294283392:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768172.211284:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768172.211286:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768172.211287:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768172.211289:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768172.211291:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1ec00. 02000000:00000001:2.0:1775768172.211293:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768172.211296:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768172.211298:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768172.211302:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768172.211304:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294283392, portal 4 00000100:00000001:2.0:1775768172.211307:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768172.211308:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606121984 : -112966103429632 : ffff994207888e00) 00000100:00000040:2.0:1775768172.211313:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207888e00 x1862022294283392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768197 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768172.211319:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768172.211321:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768172.211323:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768172.211325:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294283392, offset 0 00000400:00000200:2.0:1775768172.211328:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768172.211331:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768172.211334:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768172.211337:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768172.211339:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768172.211341:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768172.211343:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768172.211346:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225387:225387:256:4294967295] 192.168.202.156@tcp LPNI seq info [225383:225383:8:4294967295] 00000400:00000200:2.0:1775768172.211349:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768172.211355:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768172.211359:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768172.211362:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768172.211366:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768172.211368:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775768172.211372:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768172.211375:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1775768172.211378:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768172.211380:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768172.211383:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768172.211386:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768172.211388:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768172.211390:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768172.211397:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768172.211401:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207888e00 x1862022294283392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768197 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768172.211411:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768172.211414:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207888e00 x1862022294283392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768197 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768172.211418:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768172.211419:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768172.211422:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768172.211425:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:2.0:1775768172.211428:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768172.211429:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300097062 00000100:00000001:2.0:1775768172.211431:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300098366 00000100:00000001:2.0:1775768172.211433:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300097062 00000100:00000001:2.0:1775768172.211435:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300098366 00000100:00000001:2.0:1775768172.211437:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768172.211438:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768172.211440:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768172.211443:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768172/real 1775768172] req@ffff994207888e00 x1862022294283392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768172.211449:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768172.211450:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768172.211452:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768172.211453:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768172.211455:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768172.211456:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768172.211459:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207888e00 x1862022294283392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768172.211466:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207888e00 x1862022294283392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768172.211472:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768172.211473:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768172.211475:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768172.211476:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768172.211479:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207888e00 x1862022294283392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768172.211485:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768172.211488:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207888e00 x1862022294283392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768197 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768172.211493:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768172.211495:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768172.211496:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768172.211497:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768172.211499:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768172.211502:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768172.211504:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768172.211507:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768172.211509:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768172.211511:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207888e00 x1862022294283392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768197 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768172.211518:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207888e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294283392:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768172.211521:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768172.211523:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768172.211526:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207889f80 x1862022294282880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768197 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768172.211532:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768172.211534:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768172.211536:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1e400. 02000000:00000001:2.0:1775768172.211540:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768172.211542:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768172.211545:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:2.0:1775768172.211547:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768172.211549:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543f800. 02000000:00000001:2.0:1775768172.211552:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768172.211554:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768172.211556:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207889f80. 00000100:00000001:2.0:1775768172.211559:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768172.211560:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768172.211561:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768172.211564:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff99420788b100 x1862022294283008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768197 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768172.211569:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768172.211570:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768172.211572:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1d000. 02000000:00000001:2.0:1775768172.211575:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768172.211575:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768172.211576:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:2.0:1775768172.211578:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768172.211579:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543fc00. 02000000:00000001:2.0:1775768172.211580:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768172.211581:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768172.211582:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff99420788b100. 00000100:00000001:2.0:1775768172.211585:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768172.211586:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768172.211587:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768172.211590:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207888700 x1862022294283136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768197 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768172.211594:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768172.211595:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768172.211597:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f400. 02000000:00000001:2.0:1775768172.211598:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768172.211599:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768172.211600:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:2.0:1775768172.211602:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768172.211603:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543e800. 02000000:00000001:2.0:1775768172.211605:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768172.211607:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768172.211608:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207888700. 00000100:00000001:2.0:1775768172.211610:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768172.211612:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768172.211613:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768172.211616:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207889880 x1862022294283264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768197 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768172.211621:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768172.211622:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768172.211623:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1fc00. 02000000:00000001:2.0:1775768172.211625:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768172.211626:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768172.211627:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:2.0:1775768172.211629:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768172.211630:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543c800. 02000000:00000001:2.0:1775768172.211632:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768172.211633:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768172.211634:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207889880. 00000100:00000001:2.0:1775768172.211636:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768172.211637:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768172.211638:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768172.211641:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207888e00 x1862022294283392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768197 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768172.211647:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768172.211648:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768172.211650:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1ec00. 02000000:00000001:2.0:1775768172.211652:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768172.211653:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768172.211655:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:2.0:1775768172.211656:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768172.211657:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543dc00. 02000000:00000001:2.0:1775768172.211659:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768172.211660:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768172.211661:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207888e00. 00000100:00000001:2.0:1775768172.211663:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768172.211664:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768172.211667:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768172.211675:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768172.211676:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768172.211678:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768172.211680:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768172.273263:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768172.273270:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768172.273272:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768172.273275:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768172.273281:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768172.273283:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768172.273286:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768172.273288:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768172.273292:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768172.273293:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768172.273296:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768172.273297:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768172.273300:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768172.273301:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768172.273304:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768172.273305:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768172.273352:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768172.273356:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768172.273359:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768172.337214:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768172.337218:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768172.337219:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768172.337223:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768172.337238:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768172.337241:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768172.337242:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768172.337244:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768172.337246:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768173.233270:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768173.233271:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768173.233272:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768173.233276:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768173.233277:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768173.233277:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768173.233279:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768173.233281:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768173.233281:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768173.233285:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:1.0:1775768173.233287:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775768173.233287:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775768173.233288:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768173.233293:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768173.233294:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768173.233294:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768173.233295:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768173.233298:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768173.233299:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768173.233299:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768173.233305:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768173.233306:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768173.233308:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768173.233312:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768173.233312:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768173.233315:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768173.233318:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768173.297261:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768173.297266:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768173.297271:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768173.297272:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768173.297275:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768173.297276:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768173.297279:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768173.297281:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768173.297283:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768173.297285:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768173.297332:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768173.297334:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768173.297337:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768173.361275:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768173.361282:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768173.361284:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768173.361288:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768173.361292:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768173.361296:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768173.361298:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768173.361301:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768173.361304:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768173.553258:0:15036:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4300098366, at 4300098368 00000100:00000040:1.0:1775768173.553258:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4300098366, at 4300098368 00000100:00000010:1.0:1775768173.553272:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d6480. 00000100:00000010:0.0:1775768173.553275:0:15036:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d6300. 00000100:00000001:0.0:1775768174.257277:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768174.257277:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768174.257277:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768174.257289:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768174.257289:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768174.257289:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768174.257294:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768174.257294:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768174.257299:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768174.257299:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775768174.257299:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:0.0:1775768174.257311:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768174.257312:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768174.257312:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768174.257317:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768174.257320:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768174.257320:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768174.257321:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768174.257325:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768174.257328:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768174.257331:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768174.257331:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768174.257332:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768174.257335:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768174.257335:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768174.257338:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768174.257340:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768174.321293:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768174.321299:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768174.321304:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768174.321305:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768174.321308:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768174.321310:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768174.321313:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768174.321314:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768174.321319:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768174.321320:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768174.321397:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768174.321401:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768174.321404:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768174.385284:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768174.385291:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768174.385293:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768174.385298:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768174.385302:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768174.385306:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768174.385308:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768174.385311:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768174.385313:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768175.281233:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768175.281233:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768175.281239:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768175.281240:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768175.281242:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768175.281242:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768175.281248:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775768175.281248:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768175.281254:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768175.281254:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768175.281257:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768175.281257:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768175.281261:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768175.281262:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768175.281263:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768175.281266:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768175.281267:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768175.281267:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768175.281268:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768175.281270:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768175.281272:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768175.281273:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768175.281275:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768175.281278:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768175.281280:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768175.281281:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768175.281283:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768175.345245:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768175.345249:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768175.345253:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768175.345254:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768175.345256:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768175.345257:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768175.345259:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768175.345260:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768175.345262:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768175.345263:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768175.345303:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768175.345306:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768175.345308:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768175.409244:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768175.409248:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768175.409250:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768175.409254:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768175.409256:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768175.409259:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768175.409261:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768175.409262:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768175.409264:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768176.305257:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768176.305257:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768176.305260:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768176.305263:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768176.305263:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768176.305264:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768176.305266:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768176.305266:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768176.305266:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768176.305273:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775768176.305273:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768176.305277:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768176.305277:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:2.0:1775768176.305279:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768176.305282:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768176.305282:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768176.305283:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768176.305286:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768176.305287:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768176.305289:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768176.305291:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768176.305292:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768176.305294:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768176.305296:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768176.305297:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768176.305299:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768176.305302:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768176.369258:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768176.369262:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768176.369266:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768176.369267:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768176.369270:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768176.369271:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768176.369273:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768176.369274:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768176.369276:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768176.369276:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768176.369330:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768176.369334:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768176.369337:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768176.433277:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768176.433282:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768176.433283:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768176.433287:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768176.433290:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768176.433294:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768176.433296:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768176.433297:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768176.433299:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768177.329254:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768177.329254:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768177.329254:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768177.329260:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768177.329261:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768177.329261:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768177.329262:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768177.329264:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768177.329264:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768177.329268:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:1.0:1775768177.329269:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775768177.329269:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768177.329272:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1775768177.329273:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775768177.329274:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768177.329274:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768177.329275:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768177.329279:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768177.329279:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768177.329280:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1775768177.329281:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775768177.329284:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768177.329286:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1775768177.329287:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768177.329287:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1775768177.329289:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775768177.329290:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768177.329293:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:3.0:1775768177.329293:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768177.329294:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768177.329296:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768177.329298:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768177.329301:0:13471:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5429 00000400:00000200:0.0:1775768177.329310:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768177.329316:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768177.329319:0:13471:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768177.329321:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000080:00000001:2.0:1775768177.329322:0:36:0:(rw.c:1481:ll_writepages()) Process entered 00000100:00000001:0.0:1775768177.329325:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000080:00000001:2.0:1775768177.329326:0:36:0:(file.c:5250:cl_sync_file_range()) Process entered 00000100:00000040:0.0:1775768177.329329:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000001:2.0:1775768177.329329:0:36:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00000001:0.0:1775768177.329333:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768177.329335:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768177.329336:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000010:2.0:1775768177.329337:0:36:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994210b32630. 00000020:00000001:0.0:1775768177.329338:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768177.329342:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000010:2.0:1775768177.329343:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994209d2cc00. 00000020:00000001:0.0:1775768177.329344:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:0.0:1775768177.329347:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000010:2.0:1775768177.329349:0:36:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994209d2d000. 00000100:00000040:0.0:1775768177.329350:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768177.329353:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768177.329355:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000010:2.0:1775768177.329356:0:36:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942182d61e0. 00000100:00000001:0.0:1775768177.329357:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768177.329362:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:0.0:1775768177.329364:0:13471:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768177.329365:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:2.0:1775768177.329366:0:36:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207ccc2f0. 02000000:00000001:0.0:1775768177.329367:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768177.329370:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768177.329373:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:2.0:1775768177.329373:0:36:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994220156e40. 00000100:00000001:0.0:1775768177.329376:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775768177.329378:0:13471:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:2.0:1775768177.329380:0:36:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942077a8e20. 00000100:00000010:0.0:1775768177.329382:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fe680. 00000020:00000040:0.0:1775768177.329386:0:13471:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:0.0:1775768177.329388:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000080:00000010:2.0:1775768177.329388:0:36:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff9942087472a0. 02000000:00000001:0.0:1775768177.329390:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768177.329393:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000080:00000010:2.0:1775768177.329394:0:36:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b8000. 02000000:00000001:0.0:1775768177.329395:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768177.329399:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1775768177.329400:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994209d2d600. 02000000:00000010:0.0:1775768177.329406:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220029800. 00000008:00000010:2.0:1775768177.329406:0:36:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942077cd600. 00000100:00000001:0.0:1775768177.329412:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:2.0:1775768177.329412:0:36:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff99420876d680. 00008000:00000010:2.0:1775768177.329418:0:36:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076cf60. 00000100:00100000:0.0:1775768177.329421:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fe680 x1862022294283520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000080:00000010:2.0:1775768177.329422:0:36:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff994205066870. 00000020:00000001:2.0:1775768177.329428:0:36:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107759908408 : -112965949643208 : ffff994210b32638) 00000100:00000001:0.0:1775768177.329432:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000020:00001000:2.0:1775768177.329433:0:36:0:(cl_object.c:831:cl_env_get()) 1@ffff994210b32630 00000100:00000001:0.0:1775768177.329436:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768177.329438:0:36:0:(cl_object.c:151:cl_object_top()) ffff99420791a5f8 -> ffff99420791a5f8 00000100:00000040:0.0:1775768177.329440:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ba41ba08] to pc [ptlrpcd_rcv+-1] req@ffff9942099fe680 x1862022294283520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1775768177.329443:0:36:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:2.0:1775768177.329447:0:36:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:2.0:1775768177.329450:0:36:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x10176:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:2.0:1775768177.329454:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775768177.329455:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00000001:0.0:1775768177.329457:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775768177.329457:0:36:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768177.329461:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00020000:00000002:2.0:1775768177.329461:0:36:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x10176:0x0]io ffff9942077b8060 type 6 ignore/verify layout 1/0 00000100:00080000:0.0:1775768177.329468:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00020000:00000002:2.0:1775768177.329469:0:36:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207dfb6c8) owner 0000000000000000/ffff994201348000 00000100:00000001:0.0:1775768177.329470:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768177.329472:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768177.329475:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775768177.329476:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00020000:00000001:2.0:1775768177.329476:0:36:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00000100:00080000:0.0:1775768177.329478:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5429 00000100:00000001:1.0:1775768177.329479:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00020000:00000001:2.0:1775768177.329480:0:36:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00000100:00000001:1.0:1775768177.329482:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000400:00000200:0.0:1775768177.329484:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:2.0:1775768177.329484:0:36:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00020000:00000001:2.0:1775768177.329488:0:36:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768177.329489:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000040:1.0:1775768177.329490:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fe680 x1862022294283520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:2.0:1775768177.329492:0:36:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:0.0:1775768177.329494:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00020000:00000001:2.0:1775768177.329499:0:36:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00000100:00000040:0.0:1775768177.329500:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00020000:00000001:2.0:1775768177.329502:0:36:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768177.329504:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:2.0:1775768177.329507:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000100:00000040:0.0:1775768177.329509:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000001:2.0:1775768177.329510:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00000001:0.0:1775768177.329511:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:1.0:1775768177.329511:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000001:0.0:1775768177.329514:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00020000:00000001:2.0:1775768177.329514:0:36:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00000020:00000040:0.0:1775768177.329516:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00020000:00000002:2.0:1775768177.329517:0:36:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207dfb6c8) owner 0000000000000000/ffff994201348000 00000020:00000001:0.0:1775768177.329518:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768177.329520:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000100:00000001:1.0:1775768177.329520:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768177.329523:0:36:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768177.329524:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 02000000:00000001:1.0:1775768177.329527:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:0.0:1775768177.329528:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:2.0:1775768177.329528:0:36:0:(cl_io.c:823:cl_io_loop()) Process entered 02000000:00000001:1.0:1775768177.329531:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1775768177.329532:0:36:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000100:00000040:0.0:1775768177.329533:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 02000000:00000001:1.0:1775768177.329536:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768177.329537:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00020000:00000001:2.0:1775768177.329537:0:36:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00000100:00000040:0.0:1775768177.329541:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00020000:00200000:2.0:1775768177.329542:0:36:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00100000:1.0:1775768177.329543:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294283520:192.168.202.156@tcp:250:kworker.0 00000100:00000001:0.0:1775768177.329544:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00020000:00000002:2.0:1775768177.329548:0:36:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00000020:00000040:0.0:1775768177.329549:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00000001:1.0:1775768177.329551:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00080000:0.0:1775768177.329555:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 02000000:00000001:1.0:1775768177.329555:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00020000:00000001:2.0:1775768177.329555:0:36:0:(lov_io.c:155:lov_sub_get()) Process entered 02000000:00000001:1.0:1775768177.329557:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775768177.329558:0:36:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000100:00000001:0.0:1775768177.329559:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768177.329560:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1775768177.329562:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000020:00000001:2.0:1775768177.329562:0:36:0:(cl_object.c:762:cl_env_obtain()) Process entered 02000000:00000010:1.0:1775768177.329565:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543d000. 00000020:00000010:2.0:1775768177.329566:0:36:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994210b32090. 02000000:00000001:0.0:1775768177.329567:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768177.329568:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768177.329572:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768177.329574:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000020:00000010:2.0:1775768177.329574:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9942060d5200. 00000400:00000200:1.0:1775768177.329577:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00100000:0.0:1775768177.329578:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000020:00000010:2.0:1775768177.329580:0:36:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff9942060d4800. 00000400:00000010:1.0:1775768177.329581:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c738. 00000008:00000001:0.0:1775768177.329583:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768177.329586:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1775768177.329586:0:36:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942182d6840. 00000100:00000010:0.0:1775768177.329589:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fca80. 00000400:00000010:1.0:1775768177.329589:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf65d8. 00000008:00000010:2.0:1775768177.329592:0:36:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207cc8b28. 00000020:00000040:0.0:1775768177.329593:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:0.0:1775768177.329596:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000200:1.0:1775768177.329596:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294283520, portal 25 00020000:00000010:2.0:1775768177.329597:0:36:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994220156840. 02000000:00000001:0.0:1775768177.329599:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768177.329601:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768177.329602:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1775768177.329603:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641206400 : -112966068345216 : ffff9942099fe680) 00008000:00000010:2.0:1775768177.329603:0:36:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942077a9530. 02000000:00000001:0.0:1775768177.329604:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768177.329606:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768177.329610:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fe680 x1862022294283520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768202 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000010:2.0:1775768177.329610:0:36:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994208747738. 02000000:00000010:0.0:1775768177.329611:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002a800. 00000100:00000001:0.0:1775768177.329617:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:2.0:1775768177.329617:0:36:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077ba910. 00000100:00100000:0.0:1775768177.329624:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fca80 x1862022294283648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775768177.329624:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000010:2.0:1775768177.329624:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9942060d5a00. 00000100:00000040:1.0:1775768177.329631:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000008:00000010:2.0:1775768177.329631:0:36:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942077cd8c0. 00000400:00000010:1.0:1775768177.329636:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00020000:00000010:2.0:1775768177.329638:0:36:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff99420876e400. 00000100:00000001:0.0:1775768177.329639:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000200:1.0:1775768177.329641:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294283520, offset 0 00000100:00000001:0.0:1775768177.329643:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:2.0:1775768177.329643:0:36:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076c3f0. 00000400:00000200:1.0:1775768177.329648:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00000040:0.0:1775768177.329649:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000046a3ca2f] to pc [ptlrpcd_rcv+-1] req@ffff9942099fca80 x1862022294283648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000010:2.0:1775768177.329650:0:36:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942050663c0. 00000020:00000001:2.0:1775768177.329655:0:36:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107759906968 : -112965949644648 : ffff994210b32098) 00000400:00000200:1.0:1775768177.329658:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00001000:2.0:1775768177.329661:0:36:0:(cl_object.c:831:cl_env_get()) 1@ffff994210b32090 00000020:00000001:2.0:1775768177.329664:0:36:0:(cl_object.c:151:cl_object_top()) ffff99420791ae90 -> ffff99420791a5f8 00000100:00000001:0.0:1775768177.329665:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768177.329666:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00080000:0.0:1775768177.329669:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000001:2.0:1775768177.329670:0:36:0:(cl_io.c:125:__cl_io_init()) Process entered 00000400:00000200:1.0:1775768177.329673:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:0.0:1775768177.329674:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000020:00000001:2.0:1775768177.329674:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000100:00000001:0.0:1775768177.329677:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000020:00000001:2.0:1775768177.329678:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00080000:0.0:1775768177.329680:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1775768177.329680:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000020:00000001:2.0:1775768177.329681:0:36:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768177.329685:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:1.0:1775768177.329685:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00020000:00000001:2.0:1775768177.329687:0:36:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768177.329689:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5429 00000400:00000200:1.0:1775768177.329690:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00020000:00000001:2.0:1775768177.329693:0:36:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107621738216 : -112966087813400 : ffff99420876d6e8) 00000020:00000001:2.0:1775768177.329697:0:36:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000400:00000200:1.0:1775768177.329698:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225388:225388:256:4294967295] 192.168.202.156@tcp LPNI seq info [225384:225384:8:4294967295] 00000020:00000001:2.0:1775768177.329701:0:36:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775768177.329704:0:36:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768177.329706:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:2.0:1775768177.329708:0:36:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768177.329711:0:36:0:(cl_io.c:314:cl_io_lock()) Process entered 00020000:00000001:2.0:1775768177.329712:0:36:0:(lov_io.c:1108:lov_io_lock()) Process entered 00020000:00000001:2.0:1775768177.329714:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775768177.329715:0:36:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:2.0:1775768177.329719:0:36:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000400:00000200:1.0:1775768177.329722:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000020:00000001:2.0:1775768177.329722:0:36:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768177.329726:0:36:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775768177.329730:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768177.329733:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:2.0:1775768177.329735:0:36:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775768177.329736:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768177.329738:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768177.329741:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768177.329743:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000800:00000010:1.0:1775768177.329744:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541300. 00000100:00000001:0.0:1775768177.329745:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768177.329748:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768177.329750:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775768177.329750:0:36:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:0.0:1775768177.329752:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768177.329754:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000800:00000200:1.0:1775768177.329756:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000001:0.0:1775768177.329757:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000001:2.0:1775768177.329757:0:36:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1775768177.329759:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:0.0:1775768177.329763:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:2.0:1775768177.329763:0:36:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775768177.329764:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000020:00000001:0.0:1775768177.329768:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:2.0:1775768177.329768:0:36:0:(cl_io.c:528:cl_io_start()) Process entered 00020000:00000001:2.0:1775768177.329773:0:36:0:(lov_io.c:1114:lov_io_start()) Process entered 00000800:00000200:1.1:1775768177.329774:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000040:0.0:1775768177.329775:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00020000:00000001:2.0:1775768177.329776:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000001:0.0:1775768177.329779:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:2.0:1775768177.329780:0:36:0:(cl_io.c:528:cl_io_start()) Process entered 00000800:00000200:1.1:1775768177.329781:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000100:00000040:0.0:1775768177.329783:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000008:00000001:2.0:1775768177.329784:0:36:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000100:00000001:0.0:1775768177.329788:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000008:00000001:2.0:1775768177.329788:0:36:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000800:00000100:1.0:1775768177.329793:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000020:00000040:0.0:1775768177.329794:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000008:00000001:2.0:1775768177.329796:0:36:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000800:00000010:1.0:1775768177.329797:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541300. 00000100:00080000:0.0:1775768177.329800:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000008:00000020:2.0:1775768177.329800:0:36:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:2.0:1775768177.329803:0:36:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768177.329804:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768177.329804:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000008:00000001:2.0:1775768177.329809:0:36:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 02000000:00000001:0.0:1775768177.329810:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000008:00000001:2.0:1775768177.329812:0:36:0:(osc_cache.c:2287:osc_next_obj()) Process entered 02000000:00000001:0.0:1775768177.329814:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1775768177.329815:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000008:00000001:2.0:1775768177.329815:0:36:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107754940544 : -112965954611072 : ffff994210675880) 02000000:00000001:0.0:1775768177.329819:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000008:00000020:2.0:1775768177.329822:0:36:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994210675880 ready 0|-|- wr 1|-|+ rd 0|- 10 in flight 00000100:00000001:0.0:1775768177.329823:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:1.0:1775768177.329824:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000008:00100000:0.0:1775768177.329826:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000400:00000010:1.0:1775768177.329828:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000008:00000001:2.0:1775768177.329830:0:36:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000001:0.0:1775768177.329832:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000008:00000020:2.0:1775768177.329834:0:36:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000100:00000001:1.0:1775768177.329835:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1775768177.329836:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1775768177.329837:0:36:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:0.0:1775768177.329842:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fce00. 00000008:00000001:2.0:1775768177.329842:0:36:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000100:00000200:1.0:1775768177.329843:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fe680 x1862022294283520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768202 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000020:2.0:1775768177.329845:0:36:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994218232fc0 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000020:00000040:0.0:1775768177.329848:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000008:00000001:2.0:1775768177.329857:0:36:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1775768177.329861:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000008:00000001:2.0:1775768177.329862:0:36:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768177.329864:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768177.329865:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000020:00000001:2.0:1775768177.329866:0:36:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768177.329867:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768177.329870:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00020000:00000001:2.0:1775768177.329871:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768177.329872:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768177.329872:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fe680 x1862022294283520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768202 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:2.0:1775768177.329876:0:36:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768177.329878:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220028400. 00000020:00000001:2.0:1775768177.329882:0:36:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768177.329883:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768177.329887:0:36:0:(cl_io.c:555:cl_io_end()) Process entered 00020000:00000001:2.0:1775768177.329890:0:36:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00000100:00100000:0.0:1775768177.329891:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fce00 x1862022294283776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00020000:00000001:2.0:1775768177.329893:0:36:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000100:00000001:1.0:1775768177.329894:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768177.329896:0:36:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000001:2.0:1775768177.329899:0:36:0:(cl_io.c:563:cl_io_end()) Process leaving 00000100:00000001:1.0:1775768177.329900:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00020000:00000001:2.0:1775768177.329903:0:36:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768177.329907:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775768177.329909:0:36:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000020:00000001:2.0:1775768177.329912:0:36:0:(cl_io.c:563:cl_io_end()) Process leaving 00000100:00000001:0.0:1775768177.329915:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1775768177.329916:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:2.0:1775768177.329916:0:36:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000100:00000001:0.0:1775768177.329919:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775768177.329920:0:36:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00000100:00000001:1.0:1775768177.329921:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4300103485 00000100:00000040:0.0:1775768177.329923:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000060e5e9d1] to pc [ptlrpcd_rcv+-1] req@ffff9942099fce00 x1862022294283776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:2.0:1775768177.329924:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775768177.329927:0:36:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000100:00000010:1.0:1775768177.329930:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba420. 00000020:00000001:2.0:1775768177.329930:0:36:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:2.0:1775768177.329933:0:36:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768177.329936:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775768177.329937:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768177.329939:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775768177.329940:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00020000:00000001:2.0:1775768177.329940:0:36:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000020:00000001:2.0:1775768177.329944:0:36:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000100:00080000:0.0:1775768177.329945:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000020:00000001:2.0:1775768177.329946:0:36:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000100:00000001:0.0:1775768177.329947:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775768177.329948:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4300103485 00000100:00080000:0.0:1775768177.329950:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000020:00000001:2.0:1775768177.329950:0:36:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000100:00000010:1.0:1775768177.329953:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cbaa20. 00000100:00000001:0.0:1775768177.329954:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000020:00000001:2.0:1775768177.329954:0:36:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000100:00080000:0.0:1775768177.329958:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5429 00020000:00000001:2.0:1775768177.329958:0:36:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00000100:00000001:1.0:1775768177.329959:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00020000:00000001:2.0:1775768177.329961:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775768177.329964:0:36:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000400:00000200:0.0:1775768177.329967:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768177.329967:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1775768177.329967:0:36:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00020000:00000001:2.0:1775768177.329971:0:36:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768177.329973:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:1.0:1775768177.329973:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775768177.329976:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768177.329978:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1775768177.329978:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00020000:00000001:2.0:1775768177.329979:0:36:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000020:00000001:2.0:1775768177.329982:0:36:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000100:00000040:0.0:1775768177.329983:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000001:2.0:1775768177.329985:0:36:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768177.329987:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:1.0:1775768177.329988:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768177/real 1775768177] req@ffff9942099fe680 x1862022294283520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1775768177.329988:0:36:0:(cl_io.c:71:cl_io_fini()) Process entered 00000100:00000040:0.0:1775768177.329992:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00020000:00000001:2.0:1775768177.329992:0:36:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:2.0:1775768177.329995:0:36:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000100:00000001:0.0:1775768177.329997:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775768177.329999:0:36:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:0.0:1775768177.330002:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000001:2.0:1775768177.330002:0:36:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00000040:0.0:1775768177.330005:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00001000:2.0:1775768177.330006:0:36:0:(cl_object.c:925:cl_env_put()) 1@ffff994210b32090 00000020:00000001:0.0:1775768177.330008:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000001:1.0:1775768177.330010:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000040:0.0:1775768177.330011:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000010:2.0:1775768177.330013:0:36:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff9942060d4800. 00000400:00000001:1.0:1775768177.330014:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:0.0:1775768177.330015:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:0.0:1775768177.330018:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000400:00000010:1.0:1775768177.330018:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c738. 00000020:00000010:2.0:1775768177.330019:0:36:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942182d6840. 00000100:00000040:0.0:1775768177.330022:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768177.330024:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768177.330026:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000200:1.0:1775768177.330026:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf65d8 00000008:00000010:2.0:1775768177.330026:0:36:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207cc8b28. 00000400:00000010:1.0:1775768177.330030:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf65d8. 00000100:00000001:0.0:1775768177.330031:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00020000:00000010:2.0:1775768177.330033:0:36:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994220156840. 00000020:00000040:0.0:1775768177.330034:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00000001:1.0:1775768177.330035:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00008000:00000010:2.0:1775768177.330039:0:36:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942077a9530. 00000100:00080000:0.0:1775768177.330040:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768177.330044:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775768177.330044:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fe680 x1862022294283520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000010:2.0:1775768177.330045:0:36:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994208747738. 02000000:00000001:0.0:1775768177.330047:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768177.330051:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000010:2.0:1775768177.330051:0:36:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077ba910. 02000000:00000001:0.0:1775768177.330056:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1775768177.330057:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9942060d5200. 00000100:00000001:0.0:1775768177.330059:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775768177.330063:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000010:2.0:1775768177.330064:0:36:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942077cd8c0. 00000100:00000200:1.0:1775768177.330067:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fe680 x1862022294283520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:0.0:1775768177.330068:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:2.0:1775768177.330069:0:36:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff99420876e400. 00000100:00000001:0.0:1775768177.330070:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768177.330076:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fdc00. 00008000:00000010:2.0:1775768177.330076:0:36:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076c3f0. 00000020:00000040:0.0:1775768177.330081:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00000080:00000010:2.0:1775768177.330082:0:36:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942050663c0. 00000100:00000001:1.0:1775768177.330086:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:0.0:1775768177.330087:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000020:00000010:2.0:1775768177.330087:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9942060d5a00. 00000100:00000001:1.0:1775768177.330090:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1775768177.330091:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1775768177.330091:0:36:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994210b32090. 00000100:00000001:1.0:1775768177.330094:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768177.330096:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00020000:00000001:2.0:1775768177.330096:0:36:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00000100:00000001:1.0:1775768177.330098:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1775768177.330099:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00020000:00000001:2.0:1775768177.330099:0:36:0:(lov_io.c:694:lov_io_fini()) Process leaving 02000000:00000001:0.0:1775768177.330103:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775768177.330103:0:36:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000080:00200000:2.0:1775768177.330105:0:36:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x10176:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:1.0:1775768177.330107:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fe680 x1862022294283520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000001:2.0:1775768177.330109:0:36:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 02000000:00000010:0.0:1775768177.330110:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220029400. 00000020:00000001:2.0:1775768177.330111:0:36:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:2.0:1775768177.330115:0:36:0:(cl_object.c:925:cl_env_put()) 1@ffff994210b32630 00000020:00000010:2.0:1775768177.330119:0:36:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994209d2d000. 00000100:00000001:0.0:1775768177.330123:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768177.330125:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:2.0:1775768177.330125:0:36:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942182d61e0. 00000008:00000010:2.0:1775768177.330131:0:36:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207ccc2f0. 00000100:00100000:0.0:1775768177.330135:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fdc00 x1862022294283904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:1.0:1775768177.330135:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fe680 x1862022294283520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768202 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000010:2.0:1775768177.330136:0:36:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994220156e40. 00008000:00000010:2.0:1775768177.330142:0:36:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942077a8e20. 00000080:00000010:2.0:1775768177.330147:0:36:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff9942087472a0. 00000080:00000010:2.0:1775768177.330152:0:36:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b8000. 00000020:00000010:2.0:1775768177.330187:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994209d2cc00. 00000100:00000001:1.0:1775768177.330191:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00000010:2.0:1775768177.330192:0:36:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942077cd600. 00000100:00000001:1.0:1775768177.330198:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00020000:00000010:2.0:1775768177.330199:0:36:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff99420876d680. 00000100:00000001:0.0:1775768177.330201:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775768177.330204:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768177.330206:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:2.0:1775768177.330206:0:36:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076cf60. 00000100:00000001:1.0:1775768177.330207:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000080:00000010:2.0:1775768177.330211:0:36:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff994205066870. 00000100:00000001:1.0:1775768177.330212:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000040:0.0:1775768177.330215:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001372d9ae] to pc [ptlrpcd_rcv+-1] req@ffff9942099fdc00 x1862022294283904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768177.330217:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000010:2.0:1775768177.330217:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994209d2d600. 00000020:00000010:2.0:1775768177.330223:0:36:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994210b32630. 00000100:00080000:1.0:1775768177.330224:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000080:00000001:2.0:1775768177.330228:0:36:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:2.0:1775768177.330233:0:36:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768177.330234:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768177.330240:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768177.330241:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00080000:0.0:1775768177.330244:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768177.330248:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000040:1.0:1775768177.330249:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fe680 x1862022294283520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768202 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768177.330250:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768177.330252:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768177.330255:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768177.330262:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5429 00000100:00100000:1.0:1775768177.330263:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294283520:192.168.202.156@tcp:250:kworker.0 00000400:00000200:0.0:1775768177.330269:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768177.330270:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768177.330274:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000400:00000200:0.0:1775768177.330276:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000040:1.0:1775768177.330278:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fe680 x1862022294283520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768202 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768177.330281:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768177.330288:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768177.330290:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 00000100:00000001:0.0:1775768177.330293:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1775768177.330294:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000040:0.0:1775768177.330297:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000010:1.0:1775768177.330298:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543d000. 00000100:00000001:0.0:1775768177.330300:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 02000000:00000001:1.0:1775768177.330303:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768177.330304:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000001:1.0:1775768177.330305:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768177.330306:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:0.0:1775768177.330309:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775768177.330309:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000040:0.0:1775768177.330312:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:1.0:1775768177.330313:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 00000020:00000001:0.0:1775768177.330316:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 02000000:00000010:1.0:1775768177.330317:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029800. 00000020:00000001:0.0:1775768177.330321:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 02000000:00000001:1.0:1775768177.330322:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768177.330325:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000040:0.0:1775768177.330327:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000010:1.0:1775768177.330328:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fe680. 00000100:00000001:0.0:1775768177.330332:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775768177.330333:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000040:0.0:1775768177.330336:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768177.330336:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768177.330339:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768177.330341:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768177.330345:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00000001:1.0:1775768177.330345:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768177.330347:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768177.330350:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775768177.330352:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775768177.330353:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768177.330356:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1775768177.330356:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1775768177.330359:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768177.330359:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1775768177.330362:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768177.330364:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000040:1.0:1775768177.330365:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fca80 x1862022294283648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1775768177.330366:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:0.0:1775768177.330371:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768177.330375:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768177.330379:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fea00. 00000100:00000001:1.0:1775768177.330381:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000040:0.0:1775768177.330382:0:13471:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00000100:00000001:1.0:1775768177.330384:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768177.330385:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768177.330387:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768177.330388:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768177.330390:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768177.330391:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775768177.330391:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768177.330394:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768177.330395:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768177.330398:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002bc00. 00000100:00100000:1.0:1775768177.330400:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294283648:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768177.330402:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768177.330407:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00100000:0.0:1775768177.330409:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fea00 x1862022294284032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1775768177.330410:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768177.330412:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768177.330417:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1775768177.330421:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768177.330422:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775768177.330422:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543d000. 00000100:00000040:0.0:1775768177.330427:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008d4eaacc] to pc [ptlrpcd_rcv+-1] req@ffff9942099fea00 x1862022294284032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1775768177.330427:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768177.330435:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768177.330440:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c738. 00000100:00000001:0.0:1775768177.330441:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768177.330446:0:13471:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5439) 00000400:00000010:1.0:1775768177.330446:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf65d8. 00000100:00000200:1.0:1775768177.330451:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294283648, portal 10 00000100:00000001:1.0:1775768177.330453:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768177.330455:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641199232 : -112966068352384 : ffff9942099fca80) 00000100:00000040:1.0:1775768177.330462:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fca80 x1862022294283648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768202 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768177.330468:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768177.330471:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768177.330473:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000100:00000200:1.0:1775768177.330475:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294283648, offset 0 00000400:00000200:1.0:1775768177.330479:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768177.330482:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768177.330486:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768177.330489:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768177.330491:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768177.330494:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768177.330496:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768177.330499:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225389:225389:256:4294967295] 192.168.202.156@tcp LPNI seq info [225385:225385:8:4294967295] 00000400:00000200:1.0:1775768177.330503:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768177.330509:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768177.330513:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768177.330517:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541300. 00000800:00000200:1.0:1775768177.330521:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768177.330523:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775768177.330527:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768177.330529:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1775768177.330533:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768177.330535:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541300. 00000400:00000200:1.0:1775768177.330538:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768177.330541:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768177.330543:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000400:00000010:1.0:1775768177.330545:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000100:00000001:1.0:1775768177.330547:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768177.330550:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fca80 x1862022294283648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768202 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768177.330558:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768177.330560:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fca80 x1862022294283648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768202 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768177.330566:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768177.330567:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768177.330570:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768177.330574:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:1.0:1775768177.330577:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768177.330579:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300102163 00000100:00000001:1.0:1775768177.330581:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300103485 00000100:00000001:1.0:1775768177.330583:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300102163 00000100:00000001:1.0:1775768177.330585:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300103485 00000100:00000001:1.0:1775768177.330587:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768177.330589:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768177.330591:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768177.330594:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768177/real 1775768177] req@ffff9942099fca80 x1862022294283648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768177.330601:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768177.330603:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768177.330604:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c738. 00000400:00000200:1.0:1775768177.330606:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf65d8 00000400:00000010:1.0:1775768177.330608:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf65d8. 00000100:00000001:1.0:1775768177.330610:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768177.330613:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fca80 x1862022294283648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768177.330620:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fca80 x1862022294283648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768177.330627:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768177.330628:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768177.330630:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768177.330631:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768177.330635:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fca80 x1862022294283648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768177.330641:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768177.330644:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fca80 x1862022294283648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768202 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768177.330651:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768177.330653:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768177.330655:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768177.330656:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768177.330658:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768177.330660:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768177.330663:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768177.330666:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768177.330668:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768177.330671:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fca80 x1862022294283648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768202 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768177.330677:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294283648:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768177.330681:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768177.330685:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fce00 x1862022294283776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768177.330690:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768177.330692:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768177.330694:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768177.330695:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768177.330697:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768177.330700:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294283776:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768177.330703:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768177.330704:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768177.330706:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768177.330707:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768177.330710:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543d400. 02000000:00000001:1.0:1775768177.330712:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768177.330715:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768177.330718:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c738. 00000400:00000010:1.0:1775768177.330720:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf65d8. 00000100:00000200:1.0:1775768177.330723:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294283776, portal 10 00000100:00000001:1.0:1775768177.330725:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768177.330726:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641200128 : -112966068351488 : ffff9942099fce00) 00000100:00000040:1.0:1775768177.330731:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fce00 x1862022294283776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768202 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768177.330742:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768177.330744:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768177.330747:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000100:00000200:1.0:1775768177.330749:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294283776, offset 0 00000400:00000200:1.0:1775768177.330753:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768177.330757:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768177.330760:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768177.330763:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768177.330765:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768177.330767:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768177.330769:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768177.330772:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225390:225390:256:4294967295] 192.168.202.156@tcp LPNI seq info [225386:225386:8:4294967295] 00000400:00000200:1.0:1775768177.330776:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768177.330782:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768177.330786:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768177.330789:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541300. 00000800:00000200:1.0:1775768177.330793:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768177.330795:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775768177.330799:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768177.330802:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1775768177.330805:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768177.330807:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541300. 00000400:00000200:1.0:1775768177.330810:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768177.330813:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768177.330815:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000400:00000010:1.0:1775768177.330817:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000100:00000001:1.0:1775768177.330819:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768177.330822:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fce00 x1862022294283776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768202 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768177.330829:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768177.330831:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fce00 x1862022294283776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768202 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768177.330843:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768177.330845:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768177.330847:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768177.330850:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:1.0:1775768177.330853:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768177.330855:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300102155 00000100:00000001:1.0:1775768177.330857:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300103485 00000100:00000001:1.0:1775768177.330859:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300102155 00000100:00000001:1.0:1775768177.330861:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300103485 00000100:00000001:1.0:1775768177.330863:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768177.330865:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768177.330867:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768177.330870:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768177/real 1775768177] req@ffff9942099fce00 x1862022294283776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768177.330878:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768177.330879:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768177.330880:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c738. 00000400:00000200:1.0:1775768177.330883:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf65d8 00000400:00000010:1.0:1775768177.330884:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf65d8. 00000100:00000001:1.0:1775768177.330886:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768177.330889:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fce00 x1862022294283776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768177.330896:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fce00 x1862022294283776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768177.330902:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768177.330903:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768177.330904:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768177.330906:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768177.330909:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fce00 x1862022294283776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768177.330915:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768177.330918:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fce00 x1862022294283776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768202 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768177.330924:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768177.330926:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768177.330928:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768177.330929:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768177.330931:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768177.330933:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768177.330935:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768177.330939:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768177.330941:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768177.330945:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fce00 x1862022294283776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768202 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768177.330952:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294283776:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768177.330955:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768177.330958:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fdc00 x1862022294283904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768177.330964:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768177.330966:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768177.330968:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768177.330969:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768177.330971:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768177.330974:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294283904:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768177.330976:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768177.330978:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768177.330980:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768177.330981:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768177.330984:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543e400. 02000000:00000001:1.0:1775768177.330986:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768177.330989:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768177.330991:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c738. 00000400:00000010:1.0:1775768177.330994:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf65d8. 00000100:00000200:1.0:1775768177.330997:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294283904, portal 4 00000100:00000001:1.0:1775768177.330998:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768177.331000:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641203712 : -112966068347904 : ffff9942099fdc00) 00000100:00000040:1.0:1775768177.331004:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fdc00 x1862022294283904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768202 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768177.331011:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768177.331013:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768177.331015:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000100:00000200:1.0:1775768177.331017:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294283904, offset 0 00000400:00000200:1.0:1775768177.331021:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768177.331025:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768177.331028:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768177.331031:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768177.331033:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768177.331035:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768177.331037:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768177.331040:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225391:225391:256:4294967295] 192.168.202.156@tcp LPNI seq info [225387:225387:8:4294967295] 00000400:00000200:1.0:1775768177.331044:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768177.331050:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768177.331054:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768177.331057:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541300. 00000800:00000200:1.0:1775768177.331060:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768177.331062:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775768177.331066:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768177.331080:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1775768177.331084:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768177.331086:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541300. 00000400:00000200:1.0:1775768177.331088:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768177.331092:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768177.331095:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000400:00000010:1.0:1775768177.331097:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000100:00000001:1.0:1775768177.331099:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768177.331102:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fdc00 x1862022294283904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768202 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768177.331110:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768177.331112:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fdc00 x1862022294283904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768202 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768177.331118:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768177.331119:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768177.331122:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768177.331125:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:1.0:1775768177.331128:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768177.331130:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300102178 00000100:00000001:1.0:1775768177.331132:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300103485 00000100:00000001:1.0:1775768177.331134:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300102178 00000100:00000001:1.0:1775768177.331136:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300103485 00000100:00000001:1.0:1775768177.331138:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768177.331139:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768177.331141:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768177.331144:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768177/real 1775768177] req@ffff9942099fdc00 x1862022294283904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768177.331152:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768177.331153:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768177.331171:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c738. 00000400:00000200:1.0:1775768177.331174:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf65d8 00000400:00000010:1.0:1775768177.331176:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf65d8. 00000100:00000001:1.0:1775768177.331190:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768177.331194:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fdc00 x1862022294283904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768177.331202:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fdc00 x1862022294283904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768177.331208:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768177.331209:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768177.331211:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768177.331213:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768177.331216:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fdc00 x1862022294283904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768177.331222:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768177.331225:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fdc00 x1862022294283904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768202 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768177.331231:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768177.331233:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768177.331235:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768177.331236:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768177.331238:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768177.331241:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768177.331244:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768177.331247:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768177.331249:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768177.331253:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fdc00 x1862022294283904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768202 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768177.331260:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294283904:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768177.331263:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768177.331265:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768177.331268:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fca80 x1862022294283648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768202 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768177.331275:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768177.331276:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768177.331278:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543d000. 02000000:00000001:1.0:1775768177.331280:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768177.331282:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768177.331283:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:1.0:1775768177.331285:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768177.331287:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002a800. 02000000:00000001:1.0:1775768177.331289:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768177.331290:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768177.331292:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fca80. 00000100:00000001:1.0:1775768177.331294:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768177.331296:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768177.331297:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768177.331300:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fce00 x1862022294283776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768202 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768177.331306:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768177.331308:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768177.331309:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543d400. 02000000:00000001:1.0:1775768177.331311:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768177.331312:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768177.331313:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:1.0:1775768177.331315:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768177.331316:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028400. 02000000:00000001:1.0:1775768177.331318:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768177.331319:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768177.331320:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fce00. 00000100:00000001:1.0:1775768177.331322:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768177.331323:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768177.331325:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768177.331328:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fdc00 x1862022294283904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768202 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768177.331334:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768177.331335:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768177.331337:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543e400. 02000000:00000001:1.0:1775768177.331339:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768177.331340:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768177.331341:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:1.0:1775768177.331343:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768177.331344:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029400. 02000000:00000001:1.0:1775768177.331345:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768177.331346:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768177.331347:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fdc00. 00000100:00000001:1.0:1775768177.331349:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768177.331350:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768177.331351:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768177.331354:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768177.331356:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768177.331357:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768177.331358:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768177.331360:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768177.331363:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fea00 x1862022294284032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768177.331368:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768177.331370:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768177.331371:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768177.331373:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768177.331375:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768177.331377:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294284032:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768177.331380:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768177.331382:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768177.331383:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768177.331385:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768177.331387:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543e400. 02000000:00000001:1.0:1775768177.331389:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768177.331392:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768177.331394:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c738. 00000400:00000010:1.0:1775768177.331396:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf65d8. 00000100:00000200:1.0:1775768177.331399:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294284032, portal 4 00000100:00000001:1.0:1775768177.331401:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768177.331402:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641207296 : -112966068344320 : ffff9942099fea00) 00000100:00000040:1.0:1775768177.331407:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fea00 x1862022294284032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768202 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768177.331412:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768177.331414:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768177.331416:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000100:00000200:1.0:1775768177.331418:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294284032, offset 0 00000400:00000200:1.0:1775768177.331422:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768177.331425:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768177.331428:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768177.331431:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768177.331433:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768177.331435:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768177.331437:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768177.331440:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225392:225392:256:4294967295] 192.168.202.156@tcp LPNI seq info [225388:225388:8:4294967295] 00000400:00000200:1.0:1775768177.331443:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768177.331448:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768177.331452:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768177.331455:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541300. 00000800:00000200:1.0:1775768177.331459:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768177.331461:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775768177.331465:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768177.331467:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1775768177.331470:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768177.331472:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541300. 00000400:00000200:1.0:1775768177.331474:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768177.331478:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768177.331480:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000400:00000010:1.0:1775768177.331482:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000100:00000001:1.0:1775768177.331484:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768177.331487:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fea00 x1862022294284032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768202 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768177.331499:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768177.331502:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fea00 x1862022294284032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768202 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768177.331508:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768177.331509:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768177.331512:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768177.331515:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:1.0:1775768177.331517:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768177.331519:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300102182 00000100:00000001:1.0:1775768177.331521:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300103485 00000100:00000001:1.0:1775768177.331523:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300102182 00000100:00000001:1.0:1775768177.331525:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300103485 00000100:00000001:1.0:1775768177.331527:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768177.331528:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768177.331530:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768177.331533:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768177/real 1775768177] req@ffff9942099fea00 x1862022294284032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768177.331540:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768177.331542:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768177.331543:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c738. 00000400:00000200:1.0:1775768177.331545:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf65d8 00000400:00000010:1.0:1775768177.331547:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf65d8. 00000100:00000001:1.0:1775768177.331549:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768177.331551:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fea00 x1862022294284032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768177.331560:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fea00 x1862022294284032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768177.331566:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768177.331568:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768177.331569:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768177.331570:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768177.331574:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fea00 x1862022294284032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768177.331580:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768177.331583:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fea00 x1862022294284032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768202 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768177.331589:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768177.331591:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768177.331593:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768177.331594:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768177.331595:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768177.331598:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768177.331601:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768177.331604:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768177.331606:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768177.331610:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fea00 x1862022294284032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768202 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768177.331617:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294284032:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768177.331621:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768177.331622:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768177.331625:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fea00 x1862022294284032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768202 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768177.331630:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768177.331632:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768177.331633:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543e400. 02000000:00000001:1.0:1775768177.331635:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768177.331636:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768177.331638:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:1.0:1775768177.331639:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768177.331641:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002bc00. 02000000:00000001:1.0:1775768177.331643:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768177.331643:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768177.331645:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fea00. 00000100:00000001:1.0:1775768177.331646:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768177.331648:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768177.331649:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768177.331652:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768177.331653:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768177.331654:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768177.331655:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768177.393248:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768177.393253:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768177.393258:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768177.393260:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768177.393262:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768177.393264:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768177.393266:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768177.393267:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768177.393269:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768177.393270:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768177.393272:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768177.393273:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768177.393274:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768177.393274:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768177.393275:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768177.393276:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768177.393277:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768177.393278:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768177.393280:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768177.393280:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768177.393281:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768177.393282:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768177.393283:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768177.393284:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768177.393285:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768177.393286:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768177.393288:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768177.393289:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768177.393290:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768177.393290:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768177.393291:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768177.393292:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768177.393293:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768177.393294:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768177.393339:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768177.393343:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768177.393347:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768177.457275:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768177.457281:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768177.457283:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768177.457288:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768177.457292:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768177.457296:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768177.457298:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768177.457300:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768177.457303:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768178.353260:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768178.353260:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768178.353266:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768178.353267:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768178.353268:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768178.353271:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768178.353271:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768178.353272:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768178.353276:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768178.353278:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:1.0:1775768178.353279:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:2.0:1775768178.353280:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768178.353285:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768178.353286:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768178.353286:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768178.353288:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768178.353291:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768178.353291:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768178.353292:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768178.353296:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768178.353299:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768178.353299:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768178.353302:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768178.353304:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768178.353304:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768178.353308:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768178.353311:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768178.417268:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768178.417273:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768178.417278:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768178.417280:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768178.417284:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768178.417285:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768178.417288:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768178.417290:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768178.417294:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768178.417295:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768178.417359:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768178.417362:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768178.417365:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768178.481271:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768178.481277:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768178.481279:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768178.481283:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768178.481287:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768178.481290:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768178.481292:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768178.481294:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768178.481296:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768178.673243:0:15036:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4300103485, at 4300103488 00000100:00000040:1.0:1775768178.673245:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4300103485, at 4300103488 00000100:00000010:0.0:1775768178.673257:0:15036:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cba420. 00000100:00000010:1.0:1775768178.673263:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cbaa20. 00000100:00000001:3.0:1775768179.377247:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768179.377255:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768179.377256:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768179.377263:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768179.377267:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768179.377271:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768179.377273:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768179.377276:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768179.377279:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768179.377293:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768179.377296:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768179.377301:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768179.377301:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768179.377304:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768179.377305:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768179.377311:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:2.0:1775768179.377311:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768179.377319:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768179.377319:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768179.377324:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768179.377326:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768179.377327:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768179.377332:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768179.377335:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768179.377336:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768179.377338:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768179.377340:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768179.441258:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768179.441262:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768179.441266:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768179.441267:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768179.441270:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768179.441271:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768179.441273:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768179.441274:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768179.441276:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768179.441278:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768179.441370:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768179.441375:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768179.441379:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768179.505275:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768179.505281:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768179.505283:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768179.505289:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768179.505292:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768179.505296:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768179.505299:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768179.505301:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768179.505304:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768180.401248:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768180.401249:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768180.401255:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768180.401256:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768180.401258:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768180.401258:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768180.401265:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775768180.401266:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768180.401270:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768180.401271:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768180.401274:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768180.401279:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768180.401280:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768180.401281:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768180.401286:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768180.401290:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768180.401290:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768180.401302:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768180.401302:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768180.401302:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768180.401311:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768180.401312:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768180.401315:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768180.401318:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768180.401322:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768180.401325:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768180.401329:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768180.465306:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768180.465312:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768180.465316:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768180.465317:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768180.465319:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768180.465320:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768180.465323:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768180.465324:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768180.465327:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768180.465328:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768180.465394:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768180.465398:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768180.465404:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768180.529240:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768180.529247:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768180.529249:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768180.529256:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768180.529261:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768180.529264:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768180.529267:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768180.529270:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768180.529272:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768181.425249:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768181.425249:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768181.425249:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768181.425256:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768181.425256:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768181.425258:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768181.425258:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768181.425261:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:1.0:1775768181.425264:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775768181.425264:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768181.425265:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768181.425270:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768181.425270:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:2.0:1775768181.425271:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768181.425276:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768181.425276:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768181.425276:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768181.425280:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768181.425280:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768181.425284:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768181.425287:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768181.425288:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768181.425289:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768181.425291:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768181.425291:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768181.425295:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768181.425298:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768181.489284:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768181.489291:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768181.489294:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768181.489297:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768181.489301:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768181.489304:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768181.489306:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768181.489308:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768181.489311:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768181.489313:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768181.489317:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768181.489318:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768181.489322:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768181.489323:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768181.489326:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768181.489327:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768181.489381:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768181.489385:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768181.489389:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768181.553267:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768181.553272:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768181.553274:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768181.553279:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768181.553283:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768181.553286:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768181.553288:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768181.553290:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768181.553293:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768182.449261:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768182.449261:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768182.449261:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768182.449268:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768182.449269:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768182.449270:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768182.449271:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768182.449273:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775768182.449273:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768182.449273:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768182.449279:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:2.0:1775768182.449279:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775768182.449280:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00080000:0.0:1775768182.449282:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775768182.449286:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768182.449286:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768182.449287:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1775768182.449287:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1775768182.449292:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768182.449292:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768182.449292:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768182.449293:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768182.449298:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768182.449300:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768182.449301:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775768182.449303:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768182.449307:0:13471:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5434 00000100:00000001:3.0:1775768182.449307:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768182.449311:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768182.449316:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768182.449316:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768182.449319:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768182.449321:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768182.449327:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768182.449329:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768182.449333:0:13471:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768182.449337:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768182.449340:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768182.449342:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000080:00000001:2.0:1775768182.449342:0:36:0:(rw.c:1481:ll_writepages()) Process entered 00000100:00000001:0.0:1775768182.449345:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000080:00000001:2.0:1775768182.449346:0:36:0:(file.c:5250:cl_sync_file_range()) Process entered 00000020:00000001:0.0:1775768182.449350:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768182.449352:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:0.0:1775768182.449354:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768182.449356:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:2.0:1775768182.449357:0:36:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000001:0.0:1775768182.449360:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:0.0:1775768182.449363:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:0.0:1775768182.449365:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000010:2.0:1775768182.449366:0:36:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994210b32630. 00000100:00000001:0.0:1775768182.449367:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768182.449371:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000010:2.0:1775768182.449372:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9942060d5a00. 00000100:00000001:0.0:1775768182.449374:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768182.449377:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000020:00000010:2.0:1775768182.449378:0:36:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff9942060d5200. 00000100:00080000:0.0:1775768182.449383:0:13471:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000020:00000010:2.0:1775768182.449383:0:36:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942182d61e0. 00000100:00000001:0.0:1775768182.449386:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768182.449389:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000008:00000010:2.0:1775768182.449390:0:36:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207ccc2f0. 02000000:00000001:0.0:1775768182.449392:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768182.449395:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:2.0:1775768182.449395:0:36:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994220156e40. 00000100:00000001:0.0:1775768182.449397:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775768182.449399:0:13471:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:2.0:1775768182.449400:0:36:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942077a8e20. 00000100:00000010:0.0:1775768182.449404:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fc380. 00000080:00000010:2.0:1775768182.449407:0:36:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff9942087472a0. 00000020:00000040:0.0:1775768182.449408:0:13471:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 00000080:00000010:2.0:1775768182.449412:0:36:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b8000. 02000000:00000001:0.0:1775768182.449413:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768182.449415:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1775768182.449416:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9942060d4800. 00000100:00000001:0.0:1775768182.449418:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000008:00000010:2.0:1775768182.449420:0:36:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942077cd600. 02000000:00000001:0.0:1775768182.449421:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00020000:00000010:2.0:1775768182.449425:0:36:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff99420876d680. 02000000:00000001:0.0:1775768182.449426:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:2.0:1775768182.449428:0:36:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076cf60. 00000080:00000010:2.0:1775768182.449432:0:36:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff994205066870. 02000000:00000010:0.0:1775768182.449434:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220028000. 00000020:00000001:2.0:1775768182.449435:0:36:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107759908408 : -112965949643208 : ffff994210b32638) 00000020:00001000:2.0:1775768182.449439:0:36:0:(cl_object.c:831:cl_env_get()) 1@ffff994210b32630 00000100:00000001:0.0:1775768182.449442:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768182.449443:0:36:0:(cl_object.c:151:cl_object_top()) ffff9942078a76c0 -> ffff9942078a76c0 00000020:00000001:2.0:1775768182.449448:0:36:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:2.0:1775768182.449450:0:36:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:2.0:1775768182.449452:0:36:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000100:00100000:0.0:1775768182.449455:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fc380 x1862022294284160/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000001:2.0:1775768182.449457:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775768182.449460:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:2.0:1775768182.449463:0:36:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775768182.449470:0:36:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017a:0x0]io ffff9942077b8060 type 6 ignore/verify layout 1/0 00000100:00000001:0.0:1775768182.449473:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00020000:00000002:2.0:1775768182.449475:0:36:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207c1f9b0) owner 0000000000000000/ffff994201348000 00000100:00000001:0.0:1775768182.449480:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775768182.449481:0:36:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:2.0:1775768182.449485:0:36:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:2.0:1775768182.449488:0:36:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00000100:00000040:0.0:1775768182.449489:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002957419c] to pc [ptlrpcd_rcv+-1] req@ffff9942099fc380 x1862022294284160/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:2.0:1775768182.449491:0:36:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775768182.449496:0:36:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:2.0:1775768182.449503:0:36:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:2.0:1775768182.449506:0:36:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768182.449510:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775768182.449511:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:2.0:1775768182.449513:0:36:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00020000:00000002:2.0:1775768182.449515:0:36:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207c1f9b0) owner 0000000000000000/ffff994201348000 00000020:00000001:2.0:1775768182.449517:0:36:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768182.449520:0:36:0:(cl_io.c:823:cl_io_loop()) Process entered 00000100:00000001:0.0:1775768182.449522:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768182.449523:0:36:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000100:00080000:0.0:1775768182.449527:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00020000:00000001:2.0:1775768182.449527:0:36:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00000100:00080000:0.0:1775768182.449531:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00020000:00200000:2.0:1775768182.449532:0:36:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00000001:0.0:1775768182.449533:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768182.449536:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00020000:00000002:2.0:1775768182.449538:0:36:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00000100:00000001:0.0:1775768182.449539:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768182.449543:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5434 00000100:00000001:1.0:1775768182.449544:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00020000:00000001:2.0:1775768182.449546:0:36:0:(lov_io.c:155:lov_sub_get()) Process entered 00000100:00000001:1.0:1775768182.449548:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000400:00000200:0.0:1775768182.449550:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:2.0:1775768182.449550:0:36:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000100:00000001:1.0:1775768182.449551:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000001:2.0:1775768182.449554:0:36:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000400:00000200:0.0:1775768182.449557:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000020:00000010:2.0:1775768182.449559:0:36:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994210b32090. 00000100:00080000:0.0:1775768182.449560:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775768182.449560:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fc380 x1862022294284160/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1775768182.449566:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9942060d4c00. 00000100:00000040:0.0:1775768182.449568:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768182.449574:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768182.449578:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768182.449580:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768182.449581:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:1.0:1775768182.449582:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768182.449584:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000010:2.0:1775768182.449585:0:36:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994209d2d600. 00000020:00000040:0.0:1775768182.449586:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 02000000:00000001:1.0:1775768182.449587:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:0.0:1775768182.449590:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000001:1.0:1775768182.449591:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1775768182.449592:0:36:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942182d6840. 00000020:00000040:0.0:1775768182.449593:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 02000000:00000001:1.0:1775768182.449595:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768182.449598:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000008:00000010:2.0:1775768182.449601:0:36:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207cc8b28. 00000100:00100000:1.0:1775768182.449602:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294284160:192.168.202.156@tcp:250:kworker.0 00000020:00000001:0.0:1775768182.449603:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00020000:00000010:2.0:1775768182.449606:0:36:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994220156840. 00000100:00000040:0.0:1775768182.449610:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768182.449612:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00008000:00000010:2.0:1775768182.449612:0:36:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942077a9530. 00000100:00000001:0.0:1775768182.449613:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1775768182.449617:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:0.0:1775768182.449618:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000080:00000010:2.0:1775768182.449618:0:36:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994208747738. 02000000:00000001:1.0:1775768182.449620:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768182.449624:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000080:00000010:2.0:1775768182.449624:0:36:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077ba910. 02000000:00000001:1.0:1775768182.449625:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:0.0:1775768182.449630:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00080000:0.0:1775768182.449632:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 02000000:00000010:1.0:1775768182.449632:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543e400. 00000020:00000010:2.0:1775768182.449632:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994209d2cc00. 00000100:00000001:0.0:1775768182.449638:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768182.449638:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:2.0:1775768182.449638:0:36:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942077cd8c0. 02000000:00000001:0.0:1775768182.449644:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00020000:00000010:2.0:1775768182.449644:0:36:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff99420876e400. 00008000:00000010:2.0:1775768182.449648:0:36:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076c3f0. 02000000:00000001:0.0:1775768182.449649:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1775768182.449649:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000080:00000010:2.0:1775768182.449652:0:36:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942050663c0. 02000000:00000001:0.0:1775768182.449653:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775768182.449656:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c738. 00000020:00000001:2.0:1775768182.449656:0:36:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107759906968 : -112965949644648 : ffff994210b32098) 00000100:00000001:0.0:1775768182.449657:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000020:00001000:2.0:1775768182.449661:0:36:0:(cl_object.c:831:cl_env_get()) 1@ffff994210b32090 00000008:00100000:0.0:1775768182.449662:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000020:00000001:2.0:1775768182.449664:0:36:0:(cl_object.c:151:cl_object_top()) ffff9942078a7788 -> ffff9942078a76c0 00000400:00000010:1.0:1775768182.449665:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf65d8. 00000008:00000001:0.0:1775768182.449666:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768182.449667:0:36:0:(cl_io.c:125:__cl_io_init()) Process entered 00000100:00000001:0.0:1775768182.449671:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768182.449672:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000100:00000200:1.0:1775768182.449674:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294284160, portal 25 00000020:00000001:2.0:1775768182.449675:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00000010:0.0:1775768182.449676:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fed80. 00000020:00000001:2.0:1775768182.449678:0:36:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768182.449679:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000020:00000040:0.0:1775768182.449680:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000100:00000001:1.0:1775768182.449681:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641197440 : -112966068354176 : ffff9942099fc380) 00020000:00000001:2.0:1775768182.449682:0:36:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768182.449685:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00020000:00000001:2.0:1775768182.449688:0:36:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107621738216 : -112966087813400 : ffff99420876d6e8) 02000000:00000001:0.0:1775768182.449689:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768182.449692:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fc380 x1862022294284160/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768207 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768182.449695:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000020:00000001:2.0:1775768182.449695:0:36:0:(cl_io.c:399:cl_io_iter_init()) Process entered 02000000:00000001:0.0:1775768182.449700:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000020:00000001:2.0:1775768182.449700:0:36:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768182.449704:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768182.449705:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00020000:00000001:2.0:1775768182.449706:0:36:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768182.449711:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 02000000:00000010:0.0:1775768182.449712:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002b800. 00000020:00000001:2.0:1775768182.449712:0:36:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775768182.449715:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000020:00000001:2.0:1775768182.449717:0:36:0:(cl_io.c:314:cl_io_lock()) Process entered 00000100:00000001:0.0:1775768182.449720:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775768182.449720:0:36:0:(lov_io.c:1108:lov_io_lock()) Process entered 00000100:00000200:1.0:1775768182.449721:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294284160, offset 0 00020000:00000001:2.0:1775768182.449725:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775768182.449728:0:36:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:2.0:1775768182.449730:0:36:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000100:00100000:0.0:1775768182.449734:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fed80 x1862022294284288/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775768182.449734:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000020:00000001:2.0:1775768182.449734:0:36:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768182.449739:0:36:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775768182.449742:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775768182.449744:0:36:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768182.449745:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:2.0:1775768182.449748:0:36:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:2.0:1775768182.449751:0:36:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768182.449756:0:36:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768182.449757:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000020:00000001:2.0:1775768182.449762:0:36:0:(cl_io.c:528:cl_io_start()) Process entered 00020000:00000001:2.0:1775768182.449767:0:36:0:(lov_io.c:1114:lov_io_start()) Process entered 00000400:00000200:1.0:1775768182.449769:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00020000:00000001:2.0:1775768182.449771:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000001:0.0:1775768182.449772:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000020:00000001:2.0:1775768182.449775:0:36:0:(cl_io.c:528:cl_io_start()) Process entered 00000100:00000001:0.0:1775768182.449776:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768182.449777:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000008:00000001:2.0:1775768182.449780:0:36:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000100:00000040:0.0:1775768182.449783:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000c09b4d8] to pc [ptlrpcd_rcv+-1] req@ffff9942099fed80 x1862022294284288/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:2.0:1775768182.449784:0:36:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000400:00000200:1.0:1775768182.449786:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000008:00000001:2.0:1775768182.449791:0:36:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000400:00000200:1.0:1775768182.449795:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000008:00000020:2.0:1775768182.449796:0:36:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000100:00000001:0.0:1775768182.449797:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1775768182.449799:0:36:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1775768182.449801:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768182.449804:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775768182.449806:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768182.449808:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1775768182.449813:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225393:225393:256:4294967295] 192.168.202.156@tcp LPNI seq info [225389:225389:8:4294967295] 00000008:00000001:2.0:1775768182.449813:0:36:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000100:00000001:0.0:1775768182.449815:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000008:00000001:2.0:1775768182.449817:0:36:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000100:00080000:0.0:1775768182.449822:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5434 00000008:00000001:2.0:1775768182.449822:0:36:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107884691392 : -112965824860224 : ffff994218232fc0) 00000400:00000200:1.0:1775768182.449826:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00000020:2.0:1775768182.449830:0:36:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994218232fc0 ready 0|-|- wr 1|-|+ rd 0|- 10 in flight 00000008:00000001:2.0:1775768182.449835:0:36:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:2.0:1775768182.449839:0:36:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000400:00000200:1.0:1775768182.449843:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000008:00000001:2.0:1775768182.449843:0:36:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1775768182.449848:0:36:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000800:00000200:1.0:1775768182.449852:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000008:00000020:2.0:1775768182.449853:0:36:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994210675880 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000400:00000200:0.0:1775768182.449857:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000800:00000010:1.0:1775768182.449861:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541300. 00000008:00000001:2.0:1775768182.449863:0:36:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1775768182.449867:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000008:00000001:2.0:1775768182.449868:0:36:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775768182.449872:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00080000:0.0:1775768182.449874:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000020:00000001:2.0:1775768182.449874:0:36:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775768182.449877:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00020000:00000001:2.0:1775768182.449880:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768182.449881:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00020000:00000001:2.0:1775768182.449885:0:36:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768182.449887:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:1.1:1775768182.449889:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000001:2.0:1775768182.449891:0:36:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768182.449893:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000001:2.0:1775768182.449896:0:36:0:(cl_io.c:555:cl_io_end()) Process entered 00000100:00000001:0.0:1775768182.449897:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000800:00000200:1.1:1775768182.449897:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00020000:00000001:2.0:1775768182.449900:0:36:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00000020:00000001:0.0:1775768182.449901:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00020000:00000001:2.0:1775768182.449903:0:36:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000020:00000040:0.0:1775768182.449905:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:2.0:1775768182.449905:0:36:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000001:2.0:1775768182.449908:0:36:0:(cl_io.c:563:cl_io_end()) Process leaving 00000800:00000100:1.0:1775768182.449909:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000020:00000001:0.0:1775768182.449910:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00020000:00000001:2.0:1775768182.449910:0:36:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1775768182.449912:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541300. 00000020:00000040:0.0:1775768182.449915:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00020000:00000001:2.0:1775768182.449915:0:36:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000400:00000200:1.0:1775768182.449918:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000001:2.0:1775768182.449919:0:36:0:(cl_io.c:563:cl_io_end()) Process leaving 00000020:00000001:0.0:1775768182.449921:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:2.0:1775768182.449923:0:36:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000020:00000001:0.0:1775768182.449927:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000400:00000200:1.0:1775768182.449928:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00020000:00000001:2.0:1775768182.449928:0:36:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00020000:00000001:2.0:1775768182.449932:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775768182.449935:0:36:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000100:00000040:0.0:1775768182.449937:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000200:1.0:1775768182.449938:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000020:00000001:2.0:1775768182.449938:0:36:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:2.0:1775768182.449942:0:36:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775768182.449943:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00020000:00000001:2.0:1775768182.449945:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775768182.449947:0:36:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000100:00000001:0.0:1775768182.449949:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:2.0:1775768182.449950:0:36:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:2.0:1775768182.449954:0:36:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000100:00000040:0.0:1775768182.449955:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000001:2.0:1775768182.449956:0:36:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000100:00000001:1.0:1775768182.449957:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:2.0:1775768182.449960:0:36:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000100:00000001:0.0:1775768182.449961:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00020000:00000001:2.0:1775768182.449963:0:36:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00000020:00000040:0.0:1775768182.449965:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00020000:00000001:2.0:1775768182.449966:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775768182.449969:0:36:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000100:00000200:1.0:1775768182.449970:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fc380 x1862022294284160/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768207 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1775768182.449973:0:36:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000100:00080000:0.0:1775768182.449974:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00020000:00000001:2.0:1775768182.449976:0:36:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775768182.449981:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768182.449982:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775768182.449986:0:36:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000020:00000001:2.0:1775768182.449989:0:36:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 02000000:00000001:0.0:1775768182.449990:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000020:00000001:2.0:1775768182.449991:0:36:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768182.449995:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1775768182.449995:0:36:0:(cl_io.c:71:cl_io_fini()) Process entered 02000000:00000001:0.0:1775768182.449999:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775768182.449999:0:36:0:(lov_io.c:675:lov_io_fini()) Process entered 00000100:00000001:1.0:1775768182.450000:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00020000:00000001:2.0:1775768182.450002:0:36:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000100:00000001:0.0:1775768182.450004:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000020:00000001:2.0:1775768182.450004:0:36:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:2.0:1775768182.450006:0:36:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00100000:0.0:1775768182.450007:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000020:00001000:2.0:1775768182.450009:0:36:0:(cl_object.c:925:cl_env_put()) 1@ffff994210b32090 00000008:00000001:0.0:1775768182.450010:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768182.450010:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fc380 x1862022294284160/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768207 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1775768182.450014:0:36:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994209d2d600. 00000100:00000001:0.0:1775768182.450015:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1775768182.450021:0:36:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942182d6840. 00000100:00000010:0.0:1775768182.450022:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fd180. 00000100:00000001:1.0:1775768182.450025:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1775768182.450026:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000100:00000001:1.0:1775768182.450027:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000008:00000010:2.0:1775768182.450027:0:36:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207cc8b28. 02000000:00000001:0.0:1775768182.450032:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00020000:00000010:2.0:1775768182.450033:0:36:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994220156840. 00000100:00000001:1.0:1775768182.450035:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768182.450036:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:2.0:1775768182.450037:0:36:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942077a9530. 00000100:00000001:0.0:1775768182.450042:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:1.0:1775768182.450043:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000080:00000010:2.0:1775768182.450044:0:36:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994208747738. 02000000:00000001:0.0:1775768182.450046:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1775768182.450049:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4300108605 00000080:00000010:2.0:1775768182.450049:0:36:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077ba910. 02000000:00000001:0.0:1775768182.450050:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1775768182.450055:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9942060d4c00. 00000100:00000010:1.0:1775768182.450057:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cbaa20. 02000000:00000010:0.0:1775768182.450059:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002bc00. 00000008:00000010:2.0:1775768182.450059:0:36:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942077cd8c0. 00000100:00000001:1.0:1775768182.450064:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00020000:00000010:2.0:1775768182.450066:0:36:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff99420876e400. 00000100:00000001:0.0:1775768182.450067:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768182.450070:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4300108605 00008000:00000010:2.0:1775768182.450070:0:36:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076c3f0. 00000080:00000010:2.0:1775768182.450075:0:36:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942050663c0. 00000100:00100000:0.0:1775768182.450077:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fd180 x1862022294284416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000010:1.0:1775768182.450077:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba4e0. 00000020:00000010:2.0:1775768182.450080:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994209d2cc00. 00000100:00000001:1.0:1775768182.450083:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000020:00000010:2.0:1775768182.450085:0:36:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994210b32090. 00000100:00000001:1.0:1775768182.450088:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:2.0:1775768182.450089:0:36:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:2.0:1775768182.450092:0:36:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000100:00000001:1.0:1775768182.450094:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775768182.450096:0:36:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000100:00000001:1.0:1775768182.450099:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000080:00200000:2.0:1775768182.450099:0:36:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:2.0:1775768182.450107:0:36:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000100:00100000:1.0:1775768182.450108:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768182/real 1775768182] req@ffff9942099fc380 x1862022294284160/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768182.450109:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000020:00000001:2.0:1775768182.450110:0:36:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:2.0:1775768182.450114:0:36:0:(cl_object.c:925:cl_env_put()) 1@ffff994210b32630 00000100:00000001:0.0:1775768182.450115:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1775768182.450120:0:36:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff9942060d5200. 00000020:00000010:2.0:1775768182.450126:0:36:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942182d61e0. 00000100:00000040:0.0:1775768182.450127:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004ec16d17] to pc [ptlrpcd_rcv+-1] req@ffff9942099fd180 x1862022294284416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000010:2.0:1775768182.450144:0:36:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207ccc2f0. 00020000:00000010:2.0:1775768182.450151:0:36:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994220156e40. 00000400:00000001:1.0:1775768182.450178:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768182.450179:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768182.450182:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c738. 00008000:00000010:2.0:1775768182.450183:0:36:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942077a8e20. 00000400:00000200:1.0:1775768182.450189:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf65d8 00000100:00000001:0.0:1775768182.450190:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:2.0:1775768182.450191:0:36:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff9942087472a0. 00000400:00000010:1.0:1775768182.450193:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf65d8. 00000100:00080000:0.0:1775768182.450196:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000080:00000010:2.0:1775768182.450197:0:36:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b8000. 00000100:00000001:1.0:1775768182.450199:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000010:2.0:1775768182.450202:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9942060d5a00. 00000100:00080000:0.0:1775768182.450203:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000200:1.0:1775768182.450206:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fc380 x1862022294284160/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768182.450207:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000008:00000010:2.0:1775768182.450207:0:36:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942077cd600. 00000100:00080000:0.0:1775768182.450210:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00020000:00000010:2.0:1775768182.450213:0:36:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff99420876d680. 00000100:00000001:0.0:1775768182.450217:0:13471:0:(import.c:484:import_select_connection()) Process entered 00008000:00000010:2.0:1775768182.450218:0:36:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076cf60. 00000100:00080000:0.0:1775768182.450224:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5434 00000080:00000010:2.0:1775768182.450224:0:36:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff994205066870. 00000020:00000010:2.0:1775768182.450230:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9942060d4800. 00000100:00000200:1.0:1775768182.450231:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fc380 x1862022294284160/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775768182.450235:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000010:2.0:1775768182.450236:0:36:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994210b32630. 00000080:00000001:2.0:1775768182.450242:0:36:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1775768182.450243:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:1.0:1775768182.450248:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000080:00000001:2.0:1775768182.450248:0:36:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768182.450249:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1775768182.450252:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768182.450255:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768182.450256:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000080:00000001:2.0:1775768182.450257:0:36:0:(rw.c:1481:ll_writepages()) Process entered 00000100:00000001:1.0:1775768182.450259:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:2.0:1775768182.450260:0:36:0:(file.c:5250:cl_sync_file_range()) Process entered 00000100:00000001:0.0:1775768182.450261:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:2.0:1775768182.450264:0:36:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00000040:0.0:1775768182.450267:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00100000:1.0:1775768182.450268:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fc380 x1862022294284160/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1775768182.450269:0:36:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994210b32630. 00000100:00000001:0.0:1775768182.450272:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000010:2.0:1775768182.450274:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994209d2cc00. 00000020:00000001:0.0:1775768182.450279:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000010:2.0:1775768182.450280:0:36:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994209d2d600. 00000020:00000040:0.0:1775768182.450283:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000010:2.0:1775768182.450285:0:36:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942182d61e0. 00000020:00000001:0.0:1775768182.450287:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768182.450291:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000100:00000001:1.0:1775768182.450291:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000008:00000010:2.0:1775768182.450291:0:36:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207ccc2f0. 00020000:00000010:2.0:1775768182.450296:0:36:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994220156e40. 00000020:00000001:0.0:1775768182.450297:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00008000:00000010:2.0:1775768182.450300:0:36:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942077a8e20. 00000020:00000001:0.0:1775768182.450302:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:1.0:1775768182.450302:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fc380 x1862022294284160/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768207 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000010:2.0:1775768182.450305:0:36:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff9942087472a0. 00000100:00000040:0.0:1775768182.450309:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000080:00000010:2.0:1775768182.450310:0:36:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b8000. 00000100:00000001:0.0:1775768182.450314:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000010:2.0:1775768182.450316:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994209d2d000. 00000100:00000040:0.0:1775768182.450319:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000008:00000010:2.0:1775768182.450322:0:36:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942077cd600. 00000100:00000001:0.0:1775768182.450324:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00020000:00000010:2.0:1775768182.450327:0:36:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff99420876d680. 00000020:00000040:0.0:1775768182.450329:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00008000:00000010:2.0:1775768182.450332:0:36:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076cf60. 00000100:00080000:0.0:1775768182.450335:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775768182.450337:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000080:00000010:2.0:1775768182.450337:0:36:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff994205066870. 00000100:00000001:0.0:1775768182.450338:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768182.450342:0:36:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107759908408 : -112965949643208 : ffff994210b32638) 02000000:00000001:0.0:1775768182.450343:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1775768182.450345:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00001000:2.0:1775768182.450346:0:36:0:(cl_object.c:831:cl_env_get()) 1@ffff994210b32630 02000000:00000001:0.0:1775768182.450347:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768182.450349:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:2.0:1775768182.450350:0:36:0:(cl_object.c:151:cl_object_top()) ffff9942078a7f58 -> ffff9942078a7f58 02000000:00000001:0.0:1775768182.450352:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768182.450353:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1775768182.450353:0:36:0:(cl_io.c:125:__cl_io_init()) Process entered 00000100:00000001:0.0:1775768182.450356:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000080:00000001:2.0:1775768182.450357:0:36:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000100:00000001:1.0:1775768182.450358:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000008:00100000:0.0:1775768182.450359:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000080:00200000:2.0:1775768182.450359:0:36:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017b:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000100:00000001:1.0:1775768182.450361:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00000001:0.0:1775768182.450365:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768182.450367:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000100:00080000:1.0:1775768182.450369:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:0.0:1775768182.450370:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768182.450370:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:2.0:1775768182.450373:0:36:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768182.450376:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ff480. 00000100:00080000:1.0:1775768182.450376:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00020000:00000002:2.0:1775768182.450377:0:36:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017b:0x0]io ffff9942077b8060 type 6 ignore/verify layout 1/0 00000020:00000040:0.0:1775768182.450382:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00020000:00000002:2.0:1775768182.450384:0:36:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207c1f000) owner 0000000000000000/ffff994201348000 00000100:00000001:1.0:1775768182.450385:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:0.0:1775768182.450386:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00020000:00000001:2.0:1775768182.450388:0:36:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 02000000:00000001:0.0:1775768182.450389:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775768182.450391:0:36:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00000100:00000001:0.0:1775768182.450392:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00020000:00000001:2.0:1775768182.450394:0:36:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 02000000:00000001:0.0:1775768182.450395:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:1.0:1775768182.450397:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fc380 x1862022294284160/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768207 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:2.0:1775768182.450397:0:36:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768182.450398:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775768182.450401:0:36:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000010:0.0:1775768182.450404:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220029400. 00020000:00000001:2.0:1775768182.450405:0:36:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:2.0:1775768182.450408:0:36:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768182.450410:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768182.450412:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775768182.450415:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:2.0:1775768182.450418:0:36:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00000100:00100000:0.0:1775768182.450419:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ff480 x1862022294284544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00020000:00000002:2.0:1775768182.450420:0:36:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207c1f000) owner 0000000000000000/ffff994201348000 00000020:00000001:2.0:1775768182.450425:0:36:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768182.450430:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294284160:192.168.202.156@tcp:250:kworker.0 00000020:00000001:2.0:1775768182.450430:0:36:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:2.0:1775768182.450433:0:36:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00020000:00000001:2.0:1775768182.450435:0:36:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00000100:00000001:0.0:1775768182.450437:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00020000:00200000:2.0:1775768182.450439:0:36:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00000001:0.0:1775768182.450442:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768182.450442:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00020000:00000002:2.0:1775768182.450443:0:36:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00000100:00000001:1.0:1775768182.450448:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00020000:00000001:2.0:1775768182.450449:0:36:0:(lov_io.c:155:lov_sub_get()) Process entered 00000100:00000040:0.0:1775768182.450453:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e73dcd12] to pc [ptlrpcd_rcv+-1] req@ffff9942099ff480 x1862022294284544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:2.0:1775768182.450453:0:36:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000100:00000040:1.0:1775768182.450454:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fc380 x1862022294284160/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768207 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1775768182.450456:0:36:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775768182.450461:0:36:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994210b32090. 00000020:00000010:2.0:1775768182.450468:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9942060d4800. 00000020:00000010:2.0:1775768182.450475:0:36:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff9942060d5a00. 00000020:00000010:2.0:1775768182.450481:0:36:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942182d6840. 00000100:00000001:1.0:1775768182.450482:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768182.450486:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000008:00000010:2.0:1775768182.450488:0:36:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207cc8b28. 00000100:00000001:0.0:1775768182.450489:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775768182.450489:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543e400. 00020000:00000010:2.0:1775768182.450494:0:36:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994220156840. 00000100:00080000:0.0:1775768182.450496:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1775768182.450496:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00008000:00000010:2.0:1775768182.450498:0:36:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942077a9530. 00000020:00000001:1.0:1775768182.450499:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768182.450502:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000080:00000010:2.0:1775768182.450505:0:36:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994208747738. 00000100:00080000:0.0:1775768182.450506:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000020:00000001:1.0:1775768182.450506:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768182.450510:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028000. 00000100:00000001:0.0:1775768182.450511:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000080:00000010:2.0:1775768182.450511:0:36:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077ba910. 00000100:00080000:0.0:1775768182.450515:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:1.0:1775768182.450515:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768182.450517:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000020:00000010:2.0:1775768182.450517:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9942060d5200. 00000100:00000010:1.0:1775768182.450521:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fc380. 00000100:00000001:0.0:1775768182.450523:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000008:00000010:2.0:1775768182.450523:0:36:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942077cd8c0. 00000100:00000001:1.0:1775768182.450524:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768182.450527:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1775768182.450528:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5434 00020000:00000010:2.0:1775768182.450529:0:36:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff99420876e400. 00000100:00000001:1.0:1775768182.450531:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00008000:00000010:2.0:1775768182.450536:0:36:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076c3f0. 00000400:00000200:0.0:1775768182.450537:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768182.450538:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768182.450541:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768182.450542:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:1.0:1775768182.450544:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000080:00000010:2.0:1775768182.450544:0:36:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942050663c0. 00000100:00080000:0.0:1775768182.450547:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1775768182.450548:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1775768182.450549:0:36:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107759906968 : -112965949644648 : ffff994210b32098) 00000100:00000001:1.0:1775768182.450552:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768182.450554:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00001000:2.0:1775768182.450555:0:36:0:(cl_object.c:831:cl_env_get()) 1@ffff994210b32090 00000100:00000001:0.0:1775768182.450559:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768182.450559:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fed80 x1862022294284288/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1775768182.450560:0:36:0:(cl_object.c:151:cl_object_top()) ffff9942078a75f8 -> ffff9942078a7f58 00000100:00000040:0.0:1775768182.450565:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000001:2.0:1775768182.450566:0:36:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:2.0:1775768182.450570:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000100:00000001:0.0:1775768182.450571:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775768182.450574:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:0.0:1775768182.450576:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000001:2.0:1775768182.450577:0:36:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1775768182.450579:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00020000:00000001:2.0:1775768182.450582:0:36:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768182.450584:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:1.0:1775768182.450584:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00020000:00000001:2.0:1775768182.450585:0:36:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107621738216 : -112966087813400 : ffff99420876d6e8) 00000020:00000040:0.0:1775768182.450588:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000100:00000001:1.0:1775768182.450588:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768182.450589:0:36:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000020:00000001:2.0:1775768182.450592:0:36:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768182.450594:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 02000000:00000001:1.0:1775768182.450595:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00020000:00000001:2.0:1775768182.450595:0:36:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768182.450598:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1775768182.450600:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:2.0:1775768182.450601:0:36:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768182.450602:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768182.450606:0:36:0:(cl_io.c:314:cl_io_lock()) Process entered 00000100:00000040:0.0:1775768182.450608:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00020000:00000001:2.0:1775768182.450609:0:36:0:(lov_io.c:1108:lov_io_lock()) Process entered 00000100:00100000:1.0:1775768182.450610:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294284288:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768182.450611:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00020000:00000001:2.0:1775768182.450612:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775768182.450614:0:36:0:(cl_io.c:314:cl_io_lock()) Process entered 00000100:00000040:0.0:1775768182.450617:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000001:2.0:1775768182.450617:0:36:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000100:00000001:1.0:1775768182.450618:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000001:2.0:1775768182.450619:0:36:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768182.450622:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1775768182.450623:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775768182.450623:0:36:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768182.450626:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1775768182.450629:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 02000000:00000001:1.0:1775768182.450630:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00020000:00000001:2.0:1775768182.450630:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775768182.450634:0:36:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775768182.450635:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543e400. 00000100:00080000:0.0:1775768182.450636:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000020:00000001:2.0:1775768182.450639:0:36:0:(cl_io.c:286:cl_lockset_lock()) Process entered 02000000:00000001:1.0:1775768182.450640:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768182.450641:0:36:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768182.450642:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768182.450646:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000020:00000001:2.0:1775768182.450646:0:36:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768182.450651:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1775768182.450651:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:2.0:1775768182.450651:0:36:0:(cl_io.c:528:cl_io_start()) Process entered 00020000:00000001:2.0:1775768182.450654:0:36:0:(lov_io.c:1114:lov_io_start()) Process entered 02000000:00000001:0.0:1775768182.450656:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775768182.450656:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c738. 00020000:00000001:2.0:1775768182.450657:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775768182.450659:0:36:0:(cl_io.c:528:cl_io_start()) Process entered 00000100:00000001:0.0:1775768182.450660:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00000001:2.0:1775768182.450662:0:36:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000008:00100000:0.0:1775768182.450663:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000400:00000010:1.0:1775768182.450664:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf65d8. 00000008:00000001:2.0:1775768182.450665:0:36:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000008:00000001:0.0:1775768182.450669:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1775768182.450670:0:36:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000100:00000001:0.0:1775768182.450671:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775768182.450671:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294284288, portal 10 00000008:00000020:2.0:1775768182.450673:0:36:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000100:00000001:1.0:1775768182.450676:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000008:00000001:2.0:1775768182.450677:0:36:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:0.0:1775768182.450680:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fd880. 00000100:00000001:1.0:1775768182.450680:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641208192 : -112966068343424 : ffff9942099fed80) 00000008:00000001:2.0:1775768182.450681:0:36:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000008:00000001:2.0:1775768182.450685:0:36:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000020:00000040:0.0:1775768182.450686:0:13471:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00000008:00000001:2.0:1775768182.450687:0:36:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107884688256 : -112965824863360 : ffff994218232380) 00000100:00000040:1.0:1775768182.450691:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fed80 x1862022294284288/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768207 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1775768182.450692:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000008:00000020:2.0:1775768182.450693:0:36:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994218232380 ready 0|-|- wr 1|-|+ rd 0|- 9 in flight 02000000:00000001:0.0:1775768182.450696:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768182.450701:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000008:00000001:2.0:1775768182.450704:0:36:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 02000000:00000001:0.0:1775768182.450706:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000008:00000020:2.0:1775768182.450708:0:36:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 02000000:00000001:0.0:1775768182.450711:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768182.450712:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00000001:2.0:1775768182.450712:0:36:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1775768182.450717:0:36:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000100:00000040:1.0:1775768182.450718:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 02000000:00000010:0.0:1775768182.450719:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220028400. 00000008:00000020:2.0:1775768182.450721:0:36:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994210674fc0 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000400:00000010:1.0:1775768182.450723:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000100:00000001:0.0:1775768182.450725:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775768182.450727:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294284288, offset 0 00000008:00000001:2.0:1775768182.450732:0:36:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1775768182.450737:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000008:00000001:2.0:1775768182.450737:0:36:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768182.450738:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fd880 x1862022294284672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000001:2.0:1775768182.450743:0:36:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768182.450748:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:2.0:1775768182.450749:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768182.450755:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00020000:00000001:2.0:1775768182.450755:0:36:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768182.450758:0:36:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768182.450760:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768182.450760:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000020:00000001:2.0:1775768182.450763:0:36:0:(cl_io.c:555:cl_io_end()) Process entered 00020000:00000001:2.0:1775768182.450767:0:36:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00000100:00000040:0.0:1775768182.450768:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007f8a5f3b] to pc [ptlrpcd_rcv+-1] req@ffff9942099fd880 x1862022294284672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:2.0:1775768182.450769:0:36:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000400:00000200:1.0:1775768182.450770:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:2.0:1775768182.450773:0:36:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000001:2.0:1775768182.450777:0:36:0:(cl_io.c:563:cl_io_end()) Process leaving 00000400:00000200:1.0:1775768182.450779:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00020000:00000001:2.0:1775768182.450780:0:36:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775768182.450785:0:36:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000400:00000200:1.0:1775768182.450787:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1775768182.450789:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768182.450789:0:36:0:(cl_io.c:563:cl_io_end()) Process leaving 00000020:00000001:2.0:1775768182.450793:0:36:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000100:00000040:0.0:1775768182.450795:0:13471:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5444) 00000400:00000200:1.0:1775768182.450796:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00020000:00000001:2.0:1775768182.450796:0:36:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00020000:00000001:2.0:1775768182.450800:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775768182.450803:0:36:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000400:00000200:1.0:1775768182.450805:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225394:225394:256:4294967295] 192.168.202.156@tcp LPNI seq info [225390:225390:8:4294967295] 00000020:00000001:2.0:1775768182.450806:0:36:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:2.0:1775768182.450809:0:36:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775768182.450814:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768182.450818:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:2.0:1775768182.450819:0:36:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000020:00000001:2.0:1775768182.450822:0:36:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:2.0:1775768182.450825:0:36:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000020:00000001:2.0:1775768182.450827:0:36:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000001:2.0:1775768182.450830:0:36:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:2.0:1775768182.450834:0:36:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00000400:00000200:1.0:1775768182.450837:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00020000:00000001:2.0:1775768182.450837:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775768182.450840:0:36:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000001:2.0:1775768182.450842:0:36:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00020000:00000001:2.0:1775768182.450845:0:36:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775768182.450849:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775768182.450850:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00020000:00000001:2.0:1775768182.450853:0:36:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000020:00000001:2.0:1775768182.450855:0:36:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000800:00000010:1.0:1775768182.450857:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541300. 00000020:00000001:2.0:1775768182.450857:0:36:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768182.450862:0:36:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:2.0:1775768182.450864:0:36:0:(lov_io.c:675:lov_io_fini()) Process entered 00000800:00000200:1.0:1775768182.450867:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00020000:00000001:2.0:1775768182.450867:0:36:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:2.0:1775768182.450869:0:36:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:2.0:1775768182.450872:0:36:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000800:00000200:1.0:1775768182.450874:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000020:00001000:2.0:1775768182.450875:0:36:0:(cl_object.c:925:cl_env_put()) 1@ffff994210b32090 00000020:00000010:2.0:1775768182.450880:0:36:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff9942060d5a00. 00000800:00000200:1.1:1775768182.450884:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000010:2.0:1775768182.450886:0:36:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942182d6840. 00000800:00000200:1.1:1775768182.450887:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1775768182.450893:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000008:00000010:2.0:1775768182.450893:0:36:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207cc8b28. 00000800:00000010:1.0:1775768182.450896:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541300. 00020000:00000010:2.0:1775768182.450899:0:36:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994220156840. 00000400:00000200:1.0:1775768182.450900:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00008000:00000010:2.0:1775768182.450904:0:36:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942077a9530. 00000400:00000200:1.0:1775768182.450905:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768182.450910:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000080:00000010:2.0:1775768182.450911:0:36:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994208747738. 00000400:00000010:1.0:1775768182.450912:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000100:00000001:1.0:1775768182.450918:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000080:00000010:2.0:1775768182.450918:0:36:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077ba910. 00000100:00000200:1.0:1775768182.450923:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fed80 x1862022294284288/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768207 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1775768182.450924:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9942060d4800. 00000008:00000010:2.0:1775768182.450930:0:36:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942077cd8c0. 00000100:00000001:1.0:1775768182.450934:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00020000:00000010:2.0:1775768182.450935:0:36:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff99420876e400. 00000100:00000040:1.0:1775768182.450938:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fed80 x1862022294284288/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768207 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00008000:00000010:2.0:1775768182.450941:0:36:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076c3f0. 00000080:00000010:2.0:1775768182.450946:0:36:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942050663c0. 00000020:00000010:2.0:1775768182.450952:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9942060d5200. 00000100:00000001:1.0:1775768182.450954:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1775768182.450956:0:36:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994210b32090. 00000100:00000001:1.0:1775768182.450957:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00020000:00000001:2.0:1775768182.450961:0:36:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00000100:00000001:1.0:1775768182.450964:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775768182.450964:0:36:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:2.0:1775768182.450967:0:36:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000100:00000040:1.0:1775768182.450970:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000080:00200000:2.0:1775768182.450971:0:36:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:2.0:1775768182.450977:0:36:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000400:00000200:1.0:1775768182.450979:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:2.0:1775768182.450980:0:36:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000100:00000001:1.0:1775768182.450984:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300107283 00000020:00001000:2.0:1775768182.450984:0:36:0:(cl_object.c:925:cl_env_put()) 1@ffff994210b32630 00000020:00000010:2.0:1775768182.450989:0:36:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994209d2d600. 00000100:00000001:1.0:1775768182.450990:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300108605 00000100:00000001:1.0:1775768182.450995:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300107283 00000020:00000010:2.0:1775768182.450995:0:36:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942182d61e0. 00000100:00000001:1.0:1775768182.451001:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300108605 00000008:00000010:2.0:1775768182.451001:0:36:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207ccc2f0. 00000100:00000001:1.0:1775768182.451006:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000010:2.0:1775768182.451006:0:36:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994220156e40. 00000100:00000001:1.0:1775768182.451011:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:2.0:1775768182.451012:0:36:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942077a8e20. 00000100:00000001:1.0:1775768182.451013:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000080:00000010:2.0:1775768182.451019:0:36:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff9942087472a0. 00000100:00100000:1.0:1775768182.451020:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768182/real 1775768182] req@ffff9942099fed80 x1862022294284288/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000010:2.0:1775768182.451024:0:36:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b8000. 00000020:00000010:2.0:1775768182.451030:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994209d2cc00. 00000400:00000001:1.0:1775768182.451031:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768182.451034:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000008:00000010:2.0:1775768182.451035:0:36:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942077cd600. 00000400:00000010:1.0:1775768182.451036:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c738. 00020000:00000010:2.0:1775768182.451040:0:36:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff99420876d680. 00000400:00000200:1.0:1775768182.451041:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf65d8 00000400:00000010:1.0:1775768182.451044:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf65d8. 00008000:00000010:2.0:1775768182.451045:0:36:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076cf60. 00000100:00000001:1.0:1775768182.451047:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768182.451051:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fed80 x1862022294284288/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000010:2.0:1775768182.451052:0:36:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff994205066870. 00000020:00000010:2.0:1775768182.451058:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994209d2d000. 00000020:00000010:2.0:1775768182.451064:0:36:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994210b32630. 00000100:00000200:1.0:1775768182.451065:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fed80 x1862022294284288/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000001:2.0:1775768182.451069:0:36:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768182.451073:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768182.451075:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:2.0:1775768182.451075:0:36:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768182.451079:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768182.451081:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768182.451085:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fed80 x1862022294284288/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000001:2.0:1775768182.451085:0:36:0:(rw.c:1481:ll_writepages()) Process entered 00000080:00000001:2.0:1775768182.451089:0:36:0:(file.c:5250:cl_sync_file_range()) Process entered 00000020:00000001:2.0:1775768182.451092:0:36:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00000001:1.0:1775768182.451096:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:2.0:1775768182.451097:0:36:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994210b32630. 00000100:00000040:1.0:1775768182.451099:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fed80 x1862022294284288/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768207 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1775768182.451102:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9942060d5200. 00000100:00000001:1.0:1775768182.451108:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000010:2.0:1775768182.451109:0:36:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff9942060d4800. 00000100:00000001:1.0:1775768182.451112:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:2.0:1775768182.451114:0:36:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942182d61e0. 00000100:00000001:1.0:1775768182.451115:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768182.451117:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768182.451120:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768182.451122:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00000010:2.0:1775768182.451122:0:36:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207ccc2f0. 00000100:00080000:1.0:1775768182.451124:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00020000:00000010:2.0:1775768182.451139:0:36:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994220156e40. 00000100:00080000:1.0:1775768182.451143:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00008000:00000010:2.0:1775768182.451145:0:36:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942077a8e20. 00000100:00000001:1.0:1775768182.451146:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000080:00000010:2.0:1775768182.451151:0:36:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff9942087472a0. 00000100:00000040:1.0:1775768182.451152:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fed80 x1862022294284288/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768207 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000010:2.0:1775768182.451177:0:36:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b8000. 00000020:00000010:2.0:1775768182.451184:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9942060d5a00. 00000008:00000010:2.0:1775768182.451190:0:36:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942077cd600. 00000100:00100000:1.0:1775768182.451191:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294284288:192.168.202.156@tcp:38:kworker.0 00020000:00000010:2.0:1775768182.451197:0:36:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff99420876d680. 00000100:00000001:1.0:1775768182.451198:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768182.451203:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fd180 x1862022294284416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00008000:00000010:2.0:1775768182.451204:0:36:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076cf60. 00000080:00000010:2.0:1775768182.451209:0:36:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff994205066870. 00000100:00000001:1.0:1775768182.451215:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000001:2.0:1775768182.451215:0:36:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107759908408 : -112965949643208 : ffff994210b32638) 00000100:00000001:1.0:1775768182.451217:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768182.451221:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00001000:2.0:1775768182.451221:0:36:0:(cl_object.c:831:cl_env_get()) 1@ffff994210b32630 02000000:00000001:1.0:1775768182.451224:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768182.451226:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768182.451226:0:36:0:(cl_object.c:151:cl_object_top()) ffff99420791a850 -> ffff99420791a850 00000020:00000001:2.0:1775768182.451231:0:36:0:(cl_io.c:125:__cl_io_init()) Process entered 00000100:00100000:1.0:1775768182.451232:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294284416:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768182.451236:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000080:00000001:2.0:1775768182.451236:0:36:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:2.0:1775768182.451239:0:36:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017f:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 02000000:00000001:1.0:1775768182.451240:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768182.451241:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768182.451245:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:2.0:1775768182.451246:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 02000000:00000010:1.0:1775768182.451249:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543d400. 00000020:00000001:2.0:1775768182.451249:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:2.0:1775768182.451250:0:36:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768182.451255:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775768182.451256:0:36:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017f:0x0]io ffff9942077b8060 type 6 ignore/verify layout 1/0 00000400:00000200:1.0:1775768182.451260:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768182.451264:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c738. 00020000:00000002:2.0:1775768182.451264:0:36:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207dfbd90) owner 0000000000000000/ffff994201348000 00000400:00000010:1.0:1775768182.451270:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf65d8. 00020000:00000001:2.0:1775768182.451270:0:36:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:2.0:1775768182.451273:0:36:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00000100:00000200:1.0:1775768182.451275:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294284416, portal 10 00020000:00000001:2.0:1775768182.451276:0:36:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00000100:00000001:1.0:1775768182.451278:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00020000:00000001:2.0:1775768182.451279:0:36:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768182.451280:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641201024 : -112966068350592 : ffff9942099fd180) 00020000:00000001:2.0:1775768182.451284:0:36:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1775768182.451287:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fd180 x1862022294284416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768207 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:2.0:1775768182.451289:0:36:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:2.0:1775768182.451291:0:36:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768182.451297:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775768182.451299:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:2.0:1775768182.451301:0:36:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00000100:00000001:1.0:1775768182.451303:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00020000:00000002:2.0:1775768182.451304:0:36:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207dfbd90) owner 0000000000000000/ffff994201348000 00000100:00000040:1.0:1775768182.451310:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000020:00000001:2.0:1775768182.451311:0:36:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775768182.451315:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000020:00000001:2.0:1775768182.451315:0:36:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:2.0:1775768182.451316:0:36:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00020000:00000001:2.0:1775768182.451319:0:36:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00000100:00000200:1.0:1775768182.451321:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294284416, offset 0 00020000:00200000:2.0:1775768182.451322:0:36:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00000002:2.0:1775768182.451328:0:36:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00000400:00000200:1.0:1775768182.451332:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00020000:00000001:2.0:1775768182.451333:0:36:0:(lov_io.c:155:lov_sub_get()) Process entered 00020000:00000001:2.0:1775768182.451336:0:36:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000020:00000001:2.0:1775768182.451337:0:36:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000400:00000200:1.0:1775768182.451340:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000010:2.0:1775768182.451342:0:36:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994210b32090. 00000400:00000200:1.0:1775768182.451346:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000020:00000010:2.0:1775768182.451348:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9942060d4c00. 00000400:00000200:1.0:1775768182.451353:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000010:2.0:1775768182.451354:0:36:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994209d2d000. 00000400:00000200:1.0:1775768182.451358:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000020:00000010:2.0:1775768182.451359:0:36:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942182d6840. 00000400:00000200:1.0:1775768182.451362:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768182.451365:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000008:00000010:2.0:1775768182.451365:0:36:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207cc8b28. 00000400:00000200:1.0:1775768182.451371:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225395:225395:256:4294967295] 192.168.202.156@tcp LPNI seq info [225391:225391:8:4294967295] 00020000:00000010:2.0:1775768182.451371:0:36:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994220156840. 00008000:00000010:2.0:1775768182.451376:0:36:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942077a9530. 00000400:00000200:1.0:1775768182.451377:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000080:00000010:2.0:1775768182.451382:0:36:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994208747738. 00000080:00000010:2.0:1775768182.451387:0:36:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077ba910. 00000400:00000200:1.0:1775768182.451393:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000020:00000010:2.0:1775768182.451394:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994209d2cc00. 00000008:00000010:2.0:1775768182.451399:0:36:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942077cd8c0. 00000800:00000200:1.0:1775768182.451405:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00020000:00000010:2.0:1775768182.451406:0:36:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff99420876e400. 00008000:00000010:2.0:1775768182.451411:0:36:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076c3f0. 00000800:00000010:1.0:1775768182.451413:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541300. 00000080:00000010:2.0:1775768182.451416:0:36:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942050663c0. 00000800:00000200:1.0:1775768182.451421:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000001:2.0:1775768182.451421:0:36:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107759906968 : -112965949644648 : ffff994210b32098) 00000020:00001000:2.0:1775768182.451426:0:36:0:(cl_object.c:831:cl_env_get()) 1@ffff994210b32090 00000800:00000200:1.0:1775768182.451427:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000020:00000001:2.0:1775768182.451430:0:36:0:(cl_object.c:151:cl_object_top()) ffff99420791a210 -> ffff99420791a850 00000020:00000001:2.0:1775768182.451434:0:36:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:2.0:1775768182.451438:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000800:00000200:1.1:1775768182.451438:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000001:2.0:1775768182.451441:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000800:00000200:1.1:1775768182.451442:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000020:00000001:2.0:1775768182.451444:0:36:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1775768182.451448:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00020000:00000001:2.0:1775768182.451449:0:36:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1775768182.451452:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541300. 00020000:00000001:2.0:1775768182.451453:0:36:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107621738216 : -112966087813400 : ffff99420876d6e8) 00000400:00000200:1.0:1775768182.451455:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000001:2.0:1775768182.451458:0:36:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000400:00000200:1.0:1775768182.451462:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000020:00000001:2.0:1775768182.451462:0:36:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775768182.451467:0:36:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768182.451468:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000400:00000010:1.0:1775768182.451471:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000020:00000001:2.0:1775768182.451472:0:36:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768182.451474:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:2.0:1775768182.451475:0:36:0:(cl_io.c:314:cl_io_lock()) Process entered 00020000:00000001:2.0:1775768182.451478:0:36:0:(lov_io.c:1108:lov_io_lock()) Process entered 00000100:00000200:1.0:1775768182.451480:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fd180 x1862022294284416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768207 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:2.0:1775768182.451481:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775768182.451484:0:36:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:2.0:1775768182.451486:0:36:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:2.0:1775768182.451489:0:36:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768182.451494:0:36:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768182.451496:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00020000:00000001:2.0:1775768182.451498:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768182.451499:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fd180 x1862022294284416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768207 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:2.0:1775768182.451502:0:36:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768182.451506:0:36:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:2.0:1775768182.451510:0:36:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768182.451511:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768182.451513:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:2.0:1775768182.451513:0:36:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768182.451517:0:36:0:(cl_io.c:528:cl_io_start()) Process entered 00000100:00000001:1.0:1775768182.451518:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775768182.451520:0:36:0:(lov_io.c:1114:lov_io_start()) Process entered 00000100:00000040:1.0:1775768182.451522:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00020000:00000001:2.0:1775768182.451523:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775768182.451526:0:36:0:(cl_io.c:528:cl_io_start()) Process entered 00000400:00000200:1.0:1775768182.451529:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00000001:2.0:1775768182.451530:0:36:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000100:00000001:1.0:1775768182.451531:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300107276 00000008:00000001:2.0:1775768182.451533:0:36:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000100:00000001:1.0:1775768182.451535:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300108605 00000008:00000001:2.0:1775768182.451536:0:36:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000100:00000001:1.0:1775768182.451538:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300107276 00000008:00000020:2.0:1775768182.451539:0:36:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000100:00000001:1.0:1775768182.451541:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300108605 00000008:00000001:2.0:1775768182.451542:0:36:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768182.451545:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00000001:2.0:1775768182.451547:0:36:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000100:00000001:1.0:1775768182.451548:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1775768182.451550:0:36:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000100:00000001:1.0:1775768182.451551:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000008:00000001:2.0:1775768182.451553:0:36:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107754938304 : -112965954613312 : ffff994210674fc0) 00000100:00100000:1.0:1775768182.451556:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768182/real 1775768182] req@ffff9942099fd180 x1862022294284416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000020:2.0:1775768182.451558:0:36:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994210674fc0 ready 0|-|- wr 1|-|+ rd 0|- 9 in flight 00000008:00000001:2.0:1775768182.451567:0:36:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000400:00000001:1.0:1775768182.451571:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000008:00000020:2.0:1775768182.451571:0:36:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:2.0:1775768182.451573:0:36:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:1.0:1775768182.451574:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768182.451577:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c738. 00000008:00000001:2.0:1775768182.451578:0:36:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000400:00000200:1.0:1775768182.451582:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf65d8 00000008:00000020:2.0:1775768182.451582:0:36:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994218232380 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000400:00000010:1.0:1775768182.451585:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf65d8. 00000100:00000001:1.0:1775768182.451589:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000008:00000001:2.0:1775768182.451591:0:36:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:1.0:1775768182.451592:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fd180 x1862022294284416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:2.0:1775768182.451595:0:36:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768182.451601:0:36:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775768182.451605:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fd180 x1862022294284416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:2.0:1775768182.451605:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775768182.451610:0:36:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768182.451615:0:36:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768182.451620:0:36:0:(cl_io.c:555:cl_io_end()) Process entered 00000100:00000001:1.0:1775768182.451621:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768182.451624:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00020000:00000001:2.0:1775768182.451624:0:36:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00020000:00000001:2.0:1775768182.451627:0:36:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000100:00000001:1.0:1775768182.451628:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768182.451631:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1775768182.451631:0:36:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000001:2.0:1775768182.451635:0:36:0:(cl_io.c:563:cl_io_end()) Process leaving 00000100:00100000:1.0:1775768182.451637:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fd180 x1862022294284416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:2.0:1775768182.451638:0:36:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775768182.451644:0:36:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000020:00000001:2.0:1775768182.451646:0:36:0:(cl_io.c:563:cl_io_end()) Process leaving 00000100:00000001:1.0:1775768182.451647:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1775768182.451649:0:36:0:(cl_io.c:352:cl_io_unlock()) Process entered 00020000:00000001:2.0:1775768182.451651:0:36:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00000100:00000040:1.0:1775768182.451652:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fd180 x1862022294284416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768207 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:2.0:1775768182.451653:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775768182.451656:0:36:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000020:00000001:2.0:1775768182.451658:0:36:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:2.0:1775768182.451661:0:36:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768182.451662:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00020000:00000001:2.0:1775768182.451665:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768182.451666:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00020000:00000001:2.0:1775768182.451670:0:36:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000100:00000001:1.0:1775768182.451671:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:2.0:1775768182.451674:0:36:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000100:00000001:1.0:1775768182.451675:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768182.451678:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000020:00000001:2.0:1775768182.451678:0:36:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000100:00000001:1.0:1775768182.451682:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:2.0:1775768182.451682:0:36:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000001:2.0:1775768182.451684:0:36:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000100:00080000:1.0:1775768182.451685:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00020000:00000001:2.0:1775768182.451686:0:36:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00020000:00000001:2.0:1775768182.451689:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00080000:1.0:1775768182.451692:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000020:00000001:2.0:1775768182.451693:0:36:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000001:2.0:1775768182.451695:0:36:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000100:00000001:1.0:1775768182.451697:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00020000:00000001:2.0:1775768182.451698:0:36:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775768182.451702:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768182.451703:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fd180 x1862022294284416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768207 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:2.0:1775768182.451705:0:36:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000020:00000001:2.0:1775768182.451709:0:36:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000020:00000001:2.0:1775768182.451711:0:36:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768182.451715:0:36:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:2.0:1775768182.451718:0:36:0:(lov_io.c:675:lov_io_fini()) Process entered 00000100:00100000:1.0:1775768182.451721:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294284416:192.168.202.156@tcp:38:kworker.0 00020000:00000001:2.0:1775768182.451721:0:36:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:2.0:1775768182.451724:0:36:0:(cl_io.c:71:cl_io_fini()) Process entered 00000100:00000001:1.0:1775768182.451726:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000001:2.0:1775768182.451726:0:36:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:2.0:1775768182.451730:0:36:0:(cl_object.c:925:cl_env_put()) 1@ffff994210b32090 00000100:00000040:1.0:1775768182.451732:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ff480 x1862022294284544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1775768182.451734:0:36:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994209d2d000. 00000020:00000010:2.0:1775768182.451741:0:36:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942182d6840. 00000100:00000001:1.0:1775768182.451744:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000008:00000010:2.0:1775768182.451746:0:36:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207cc8b28. 00000100:00000001:1.0:1775768182.451747:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:2.0:1775768182.451751:0:36:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994220156840. 02000000:00000001:1.0:1775768182.451752:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768182.451756:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00008000:00000010:2.0:1775768182.451757:0:36:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942077a9530. 02000000:00000001:1.0:1775768182.451762:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:2.0:1775768182.451763:0:36:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994208747738. 00000080:00000010:2.0:1775768182.451768:0:36:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077ba910. 00000100:00100000:1.0:1775768182.451770:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294284544:192.168.202.156@tcp:8:kworker.0 00000020:00000010:2.0:1775768182.451772:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9942060d4c00. 00000100:00000001:1.0:1775768182.451778:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000008:00000010:2.0:1775768182.451779:0:36:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942077cd8c0. 02000000:00000001:1.0:1775768182.451781:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00020000:00000010:2.0:1775768182.451783:0:36:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff99420876e400. 02000000:00000001:1.0:1775768182.451785:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:2.0:1775768182.451787:0:36:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076c3f0. 02000000:00000001:1.0:1775768182.451788:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000080:00000010:2.0:1775768182.451792:0:36:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942050663c0. 02000000:00000010:1.0:1775768182.451795:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543d000. 00000020:00000010:2.0:1775768182.451797:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994209d2cc00. 02000000:00000001:1.0:1775768182.451799:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1775768182.451802:0:36:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994210b32090. 00020000:00000001:2.0:1775768182.451805:0:36:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00000400:00000200:1.0:1775768182.451807:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:2.0:1775768182.451808:0:36:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:2.0:1775768182.451811:0:36:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000400:00000010:1.0:1775768182.451812:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c738. 00000080:00200000:2.0:1775768182.451813:0:36:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017f:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000400:00000010:1.0:1775768182.451817:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf65d8. 00000080:00000001:2.0:1775768182.451818:0:36:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:2.0:1775768182.451822:0:36:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:2.0:1775768182.451824:0:36:0:(cl_object.c:925:cl_env_put()) 1@ffff994210b32630 00000100:00000200:1.0:1775768182.451825:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294284544, portal 4 00000020:00000010:2.0:1775768182.451828:0:36:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff9942060d4800. 00000100:00000001:1.0:1775768182.451830:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768182.451833:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641209984 : -112966068341632 : ffff9942099ff480) 00000020:00000010:2.0:1775768182.451833:0:36:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942182d61e0. 00000008:00000010:2.0:1775768182.451838:0:36:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207ccc2f0. 00000100:00000040:1.0:1775768182.451843:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ff480 x1862022294284544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768207 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000010:2.0:1775768182.451843:0:36:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994220156e40. 00008000:00000010:2.0:1775768182.451847:0:36:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942077a8e20. 00000080:00000010:2.0:1775768182.451853:0:36:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff9942087472a0. 00000100:00000001:1.0:1775768182.451855:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768182.451858:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000080:00000010:2.0:1775768182.451858:0:36:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b8000. 00000400:00000010:1.0:1775768182.451863:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000020:00000010:2.0:1775768182.451863:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9942060d5200. 00000100:00000200:1.0:1775768182.451869:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294284544, offset 0 00000008:00000010:2.0:1775768182.451870:0:36:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942077cd600. 00020000:00000010:2.0:1775768182.451874:0:36:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff99420876d680. 00000400:00000200:1.0:1775768182.451875:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00008000:00000010:2.0:1775768182.451880:0:36:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076cf60. 00000400:00000200:1.0:1775768182.451881:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000080:00000010:2.0:1775768182.451886:0:36:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff994205066870. 00000400:00000200:1.0:1775768182.451887:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768182.451892:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000010:2.0:1775768182.451892:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9942060d5a00. 00000400:00000200:1.0:1775768182.451896:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000020:00000010:2.0:1775768182.451898:0:36:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994210b32630. 00000400:00000200:1.0:1775768182.451899:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000080:00000001:2.0:1775768182.451903:0:36:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1775768182.451904:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000080:00000001:2.0:1775768182.451908:0:36:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768182.451909:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225396:225396:256:4294967295] 192.168.202.156@tcp LPNI seq info [225392:225392:8:4294967295] 00000400:00000200:1.0:1775768182.451916:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768182.451921:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768182.451925:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768182.451927:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541300. 00000800:00000200:1.0:1775768182.451930:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768182.451932:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775768182.451936:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768182.451938:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1775768182.451940:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768182.451942:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541300. 00000400:00000200:1.0:1775768182.451945:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768182.451949:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768182.451952:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000400:00000010:1.0:1775768182.451954:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000100:00000001:1.0:1775768182.451956:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768182.451960:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ff480 x1862022294284544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768207 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768182.451969:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768182.451972:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ff480 x1862022294284544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768207 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768182.451985:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768182.451987:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768182.451991:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768182.451994:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:1.0:1775768182.451998:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768182.452000:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300107299 00000100:00000001:1.0:1775768182.452002:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300108605 00000100:00000001:1.0:1775768182.452004:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300107299 00000100:00000001:1.0:1775768182.452007:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300108605 00000100:00000001:1.0:1775768182.452008:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768182.452010:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768182.452011:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768182.452014:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768182/real 1775768182] req@ffff9942099ff480 x1862022294284544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768182.452020:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768182.452021:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768182.452022:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c738. 00000400:00000200:1.0:1775768182.452024:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf65d8 00000400:00000010:1.0:1775768182.452025:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf65d8. 00000100:00000001:1.0:1775768182.452027:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768182.452028:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ff480 x1862022294284544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768182.452034:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ff480 x1862022294284544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768182.452038:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768182.452039:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768182.452040:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768182.452041:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768182.452043:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ff480 x1862022294284544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768182.452047:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768182.452050:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ff480 x1862022294284544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768207 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768182.452054:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768182.452055:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768182.452056:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768182.452057:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768182.452058:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768182.452061:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768182.452062:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768182.452064:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768182.452065:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768182.452067:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ff480 x1862022294284544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768207 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768182.452072:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294284544:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768182.452075:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768182.452076:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768182.452078:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fed80 x1862022294284288/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768207 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768182.452082:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768182.452083:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768182.452084:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543e400. 02000000:00000001:1.0:1775768182.452086:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768182.452087:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768182.452088:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:1.0:1775768182.452089:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768182.452090:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002b800. 02000000:00000001:1.0:1775768182.452092:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768182.452093:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768182.452094:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fed80. 00000100:00000001:1.0:1775768182.452096:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768182.452097:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768182.452098:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768182.452100:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fd180 x1862022294284416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768207 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768182.452104:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768182.452105:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768182.452106:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543d400. 02000000:00000001:1.0:1775768182.452107:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768182.452107:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768182.452108:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:1.0:1775768182.452109:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768182.452110:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002bc00. 02000000:00000001:1.0:1775768182.452111:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768182.452112:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768182.452112:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fd180. 00000100:00000001:1.0:1775768182.452114:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768182.452114:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768182.452115:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768182.452117:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ff480 x1862022294284544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768207 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768182.452120:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768182.452121:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768182.452122:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543d000. 02000000:00000001:1.0:1775768182.452123:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768182.452124:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768182.452124:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:1.0:1775768182.452125:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768182.452126:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029400. 02000000:00000001:1.0:1775768182.452127:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768182.452141:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768182.452142:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ff480. 00000100:00000001:1.0:1775768182.452144:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768182.452145:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768182.452146:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768182.452149:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768182.452150:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768182.452151:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768182.452153:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768182.452154:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768182.452171:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fd880 x1862022294284672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768182.452176:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768182.452177:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768182.452178:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768182.452180:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768182.452181:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768182.452183:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294284672:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768182.452185:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768182.452186:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768182.452187:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768182.452189:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768182.452191:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543d000. 02000000:00000001:1.0:1775768182.452192:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768182.452195:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768182.452197:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c738. 00000400:00000010:1.0:1775768182.452199:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf65d8. 00000100:00000200:1.0:1775768182.452201:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294284672, portal 4 00000100:00000001:1.0:1775768182.452202:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768182.452203:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641202816 : -112966068348800 : ffff9942099fd880) 00000100:00000040:1.0:1775768182.452207:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fd880 x1862022294284672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768207 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768182.452212:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768182.452213:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768182.452215:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000100:00000200:1.0:1775768182.452216:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294284672, offset 0 00000400:00000200:1.0:1775768182.452219:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768182.452222:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768182.452224:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768182.452226:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768182.452228:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768182.452230:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768182.452232:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768182.452234:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225397:225397:256:4294967295] 192.168.202.156@tcp LPNI seq info [225393:225393:8:4294967295] 00000400:00000200:1.0:1775768182.452236:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768182.452240:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768182.452243:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768182.452245:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541300. 00000800:00000200:1.0:1775768182.452248:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768182.452250:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775768182.452253:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768182.452254:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1775768182.452257:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768182.452258:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541300. 00000400:00000200:1.0:1775768182.452260:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768182.452263:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768182.452264:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000400:00000010:1.0:1775768182.452265:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000100:00000001:1.0:1775768182.452267:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768182.452269:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fd880 x1862022294284672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768207 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768182.452274:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768182.452276:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fd880 x1862022294284672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768207 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768182.452280:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768182.452281:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768182.452283:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768182.452284:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:1.0:1775768182.452286:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768182.452288:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300107303 00000100:00000001:1.0:1775768182.452290:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300108605 00000100:00000001:1.0:1775768182.452291:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300107303 00000100:00000001:1.0:1775768182.452293:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300108605 00000100:00000001:1.0:1775768182.452295:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768182.452297:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768182.452299:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768182.452303:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768182/real 1775768182] req@ffff9942099fd880 x1862022294284672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768182.452311:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768182.452312:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768182.452314:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c738. 00000400:00000200:1.0:1775768182.452316:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf65d8 00000400:00000010:1.0:1775768182.452317:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf65d8. 00000100:00000001:1.0:1775768182.452319:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768182.452323:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fd880 x1862022294284672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768182.452332:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fd880 x1862022294284672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768182.452339:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768182.452341:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768182.452343:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768182.452344:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768182.452348:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fd880 x1862022294284672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768182.452356:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768182.452359:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fd880 x1862022294284672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768207 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768182.452364:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768182.452366:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768182.452367:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768182.452368:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768182.452369:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768182.452372:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768182.452374:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768182.452376:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768182.452377:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768182.452380:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fd880 x1862022294284672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768207 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768182.452388:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294284672:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768182.452393:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768182.452395:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768182.452398:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fd880 x1862022294284672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768207 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768182.452405:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768182.452407:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768182.452409:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543d000. 02000000:00000001:1.0:1775768182.452411:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768182.452412:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768182.452414:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:1.0:1775768182.452416:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768182.452418:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028400. 02000000:00000001:1.0:1775768182.452420:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768182.452422:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768182.452423:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fd880. 00000100:00000001:1.0:1775768182.452426:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768182.452427:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768182.452429:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768182.452433:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768182.452434:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768182.452436:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768182.452438:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768182.513289:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768182.513294:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768182.513298:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768182.513299:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768182.513302:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768182.513303:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768182.513306:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768182.513307:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768182.513309:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768182.513310:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768182.513367:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768182.513372:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768182.513376:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768182.577224:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768182.577231:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768182.577232:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768182.577237:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768182.577240:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768182.577243:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768182.577246:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768182.577248:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768182.577249:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768183.473259:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768183.473260:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768183.473260:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768183.473266:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768183.473266:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768183.473267:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768183.473268:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768183.473271:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768183.473271:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768183.473273:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775768183.473277:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:1.0:1775768183.473278:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768183.473278:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768183.473282:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768183.473283:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768183.473284:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768183.473285:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768183.473288:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768183.473288:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768183.473289:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768183.473294:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768183.473295:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768183.473299:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768183.473303:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768183.473303:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768183.473307:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768183.473310:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768183.537245:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768183.537250:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768183.537254:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768183.537256:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768183.537258:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768183.537259:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768183.537261:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768183.537262:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768183.537264:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768183.537265:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768183.537319:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768183.537323:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768183.537326:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768183.601247:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768183.601254:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768183.601256:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768183.601262:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768183.601266:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768183.601270:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768183.601273:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768183.601276:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768183.601279:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768183.793215:0:15036:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4300108605, at 4300108608 00000100:00000040:1.0:1775768183.793215:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4300108605, at 4300108608 00000100:00000010:0.0:1775768183.793230:0:15036:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cbaa20. 00000100:00000010:1.0:1775768183.793236:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cba4e0. 00000100:00000001:1.0:1775768184.497241:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768184.497241:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768184.497246:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768184.497249:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768184.497254:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768184.497256:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768184.497258:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768184.497260:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768184.497262:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768184.497265:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768184.497265:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768184.497267:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768184.497270:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768184.497271:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768184.497275:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768184.497277:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768184.497279:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768184.497283:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768184.497283:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768184.497286:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768184.497288:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768184.497292:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768184.497297:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768184.497300:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768184.497302:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768184.497304:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768184.497307:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768184.561218:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768184.561222:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768184.561227:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768184.561229:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768184.561231:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768184.561233:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768184.561236:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768184.561237:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768184.561239:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768184.561240:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768184.561294:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768184.561297:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768184.561300:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768184.625214:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768184.625219:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768184.625220:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768184.625225:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768184.625228:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768184.625232:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768184.625234:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768184.625235:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768184.625238:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768185.521253:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768185.521260:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768185.521262:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768185.521268:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768185.521272:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768185.521276:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768185.521278:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768185.521282:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768185.521286:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768185.521410:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768185.521413:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768185.521415:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768185.521417:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768185.521420:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768185.521421:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768185.521423:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768185.521425:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768185.521426:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768185.521508:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768185.521512:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768185.521513:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768185.521515:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768185.521517:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768185.521519:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768185.521521:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768185.521523:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768185.521525:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768185.585270:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768185.585276:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768185.585283:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768185.585285:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768185.585288:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768185.585290:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768185.585293:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768185.585295:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768185.585298:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768185.585299:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768185.585356:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768185.585359:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768185.585362:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768185.649272:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768185.649278:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768185.649279:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768185.649284:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768185.649287:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768185.649291:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768185.649293:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768185.649295:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768185.649296:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768186.545220:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768186.545220:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768186.545226:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768186.545226:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768186.545228:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768186.545228:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768186.545233:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:2.0:1775768186.545233:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768186.545234:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768186.545237:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768186.545237:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768186.545238:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768186.545241:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768186.545242:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768186.545242:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768186.545245:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768186.545247:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768186.545249:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768186.545250:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768186.545253:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768186.545256:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768186.545259:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775768186.545263:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768186.545266:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768186.545268:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768186.545270:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768186.545271:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768186.609266:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768186.609270:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768186.609274:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768186.609277:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768186.609279:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768186.609281:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768186.609283:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768186.609285:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768186.609286:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768186.609288:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768186.609290:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768186.609291:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768186.609292:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768186.609293:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768186.609294:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768186.609294:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768186.609295:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768186.609296:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768186.609298:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768186.609299:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768186.609300:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768186.609300:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768186.609301:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768186.609302:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768186.609303:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768186.609304:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768186.609306:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768186.609306:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768186.609307:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768186.609308:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768186.609314:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768186.609315:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768186.609316:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768186.609317:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768186.609355:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768186.609358:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768186.609360:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768186.673243:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768186.673248:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768186.673249:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768186.673253:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768186.673256:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768186.673259:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768186.673261:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768186.673262:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768186.673264:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768187.569221:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768187.569227:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768187.569229:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768187.569231:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768187.569233:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:1.0:1775768187.569233:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768187.569236:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768187.569238:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768187.569238:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768187.569239:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768187.569242:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768187.569243:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768187.569245:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00100000:0.0:1775768187.569247:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768187.569247:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775768187.569251:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768187.569252:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768187.569252:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768187.569256:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768187.569257:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768187.569261:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768187.569262:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768187.569263:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768187.569267:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768187.569268:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768187.569271:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768187.569274:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768187.569276:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768187.569281:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775768187.569283:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775768187.569286:0:36:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768187.569290:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775768187.569297:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5439 00000400:00000200:1.0:1775768187.569306:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768187.569312:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775768187.569315:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775768187.569318:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768187.569321:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768187.569324:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768187.569326:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775768187.569330:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775768187.569332:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:1.0:1775768187.569334:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775768187.569336:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:1.0:1775768187.569339:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:1.0:1775768187.569341:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:1.0:1775768187.569343:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768187.569345:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768187.569347:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768187.569349:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775768187.569351:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:1.0:1775768187.569354:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775768187.569357:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768187.569360:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775768187.569364:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768187.569366:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768187.569368:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775768187.569370:0:36:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775768187.569379:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207888e00. 00000020:00000040:1.0:1775768187.569383:0:36:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:1.0:1775768187.569386:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768187.569387:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768187.569390:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775768187.569392:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775768187.569395:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775768187.569403:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543d400. 00000100:00000001:1.0:1775768187.569410:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768187.569418:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207888e00 x1862022294284800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775768187.569428:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775768187.569432:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768187.569437:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001812f966] to pc [ptlrpcd_rcv+-1] req@ffff994207888e00 x1862022294284800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768187.569458:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768187.569461:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768187.569463:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775768187.569464:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775768187.569465:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768187.569466:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775768187.569468:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5439 00000400:00000200:1.0:1775768187.569471:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768187.569473:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775768187.569475:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775768187.569477:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768187.569479:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768187.569481:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768187.569482:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775768187.569484:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775768187.569485:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:1.0:1775768187.569486:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775768187.569487:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000100:00000001:2.0:1775768187.569487:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1775768187.569490:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000001:2.0:1775768187.569490:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:1.0:1775768187.569491:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000001:2.0:1775768187.569493:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768187.569494:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768187.569496:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768187.569500:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000040:2.0:1775768187.569500:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207888e00 x1862022294284800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768187.569503:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775768187.569509:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00000001:2.0:1775768187.569513:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:1.0:1775768187.569515:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775768187.569515:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768187.569519:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768187.569519:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768187.569521:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768187.569522:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775768187.569523:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768187.569526:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207888e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294284800:192.168.202.156@tcp:250:kworker.0 00000100:00000001:2.0:1775768187.569529:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768187.569531:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768187.569534:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768187.569536:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1775768187.569537:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768187.569540:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775768187.569541:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f000. 00000100:00000001:1.0:1775768187.569542:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:2.0:1775768187.569544:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:1.0:1775768187.569547:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775768187.569552:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768187.569553:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768187.569556:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775768187.569558:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000100:00000010:1.0:1775768187.569561:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207889880. 00000020:00000040:1.0:1775768187.569565:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000400:00000010:2.0:1775768187.569566:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 02000000:00000001:1.0:1775768187.569567:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768187.569569:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768187.569572:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775768187.569573:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775768187.569575:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1775768187.569575:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294284800, portal 25 00000100:00000001:2.0:1775768187.569579:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 02000000:00000010:1.0:1775768187.569582:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543e400. 00000100:00000001:2.0:1775768187.569582:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606121984 : -112966103429632 : ffff994207888e00) 00000100:00000001:1.0:1775768187.569587:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768187.569591:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207888e00 x1862022294284800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768212 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768187.569596:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207889880 x1862022294284928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775768187.569606:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1775768187.569610:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:2.0:1775768187.569612:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000100:00000001:1.0:1775768187.569613:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775768187.569617:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000040:1.0:1775768187.569618:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f11856b8] to pc [ptlrpcd_rcv+-1] req@ffff994207889880 x1862022294284928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768187.569622:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294284800, offset 0 00000100:00000001:1.0:1775768187.569629:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768187.569630:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768187.569635:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000200:2.0:1775768187.569635:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00000001:1.0:1775768187.569639:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775768187.569640:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768187.569642:0:36:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:2.0:1775768187.569645:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00080000:1.0:1775768187.569646:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5439 00000400:00000200:2.0:1775768187.569652:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768187.569653:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768187.569660:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000400:00000200:2.0:1775768187.569660:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:1.0:1775768187.569664:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1775768187.569667:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000100:00000040:1.0:1775768187.569670:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000200:2.0:1775768187.569673:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000100:00000001:1.0:1775768187.569674:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1775768187.569677:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00000040:1.0:1775768187.569679:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768187.569682:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000200:2.0:1775768187.569684:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225398:225398:256:4294967295] 192.168.202.156@tcp LPNI seq info [225394:225394:8:4294967295] 00000020:00000001:1.0:1775768187.569687:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775768187.569689:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000400:00000200:2.0:1775768187.569690:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:1.0:1775768187.569693:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775768187.569695:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:1.0:1775768187.569697:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:1.0:1775768187.569699:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:1.0:1775768187.569703:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768187.569705:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768187.569711:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000200:2.0:1775768187.569713:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000100:00000001:1.0:1775768187.569714:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775768187.569718:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000800:00000200:2.0:1775768187.569718:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00080000:1.0:1775768187.569724:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000800:00000010:2.0:1775768187.569724:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000100:00000001:1.0:1775768187.569729:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768187.569731:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:2.0:1775768187.569734:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 02000000:00000001:1.0:1775768187.569736:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:2.0:1775768187.569738:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 02000000:00000001:1.0:1775768187.569739:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768187.569742:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775768187.569745:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775768187.569748:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775768187.569748:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:1.0:1775768187.569752:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775768187.569754:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000100:00000010:1.0:1775768187.569757:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207888700. 00000800:00000100:2.0:1775768187.569761:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768187.569763:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000020:00000040:1.0:1775768187.569770:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000400:00000200:2.0:1775768187.569770:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000001:1.0:1775768187.569776:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768187.569778:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768187.569779:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000100:00000001:1.0:1775768187.569782:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775768187.569785:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:2.0:1775768187.569785:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 02000000:00000001:1.0:1775768187.569787:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775768187.569788:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 02000000:00000010:1.0:1775768187.569794:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543d000. 00000100:00000001:2.0:1775768187.569794:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1775768187.569799:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1775768187.569799:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207888e00 x1862022294284800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768212 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768187.569808:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207888700 x1862022294285056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775768187.569810:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768187.569815:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207888e00 x1862022294284800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768212 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768187.569819:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775768187.569822:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768187.569827:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768187.569830:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009f2ba301] to pc [ptlrpcd_rcv+-1] req@ffff994207888700 x1862022294285056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768187.569830:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768187.569834:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768187.569837:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768187.569840:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00080000:1.0:1775768187.569841:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775768187.569842:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4300113725 00000100:00080000:1.0:1775768187.569845:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775768187.569847:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775768187.569848:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000010:2.0:1775768187.569848:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d61e0. 00000100:00000001:1.0:1775768187.569852:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775768187.569854:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5439 00000100:00000001:2.0:1775768187.569855:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000400:00000200:1.0:1775768187.569859:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768187.569861:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4300113725 00000400:00000200:1.0:1775768187.569863:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000010:2.0:1775768187.569864:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d6840. 00000100:00080000:1.0:1775768187.569866:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1775768187.569867:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1775768187.569872:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1775768187.569873:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768187.569874:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768187.569878:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775768187.569878:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000040:1.0:1775768187.569883:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00100000:2.0:1775768187.569885:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768187/real 1775768187] req@ffff994207888e00 x1862022294284800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768187.569887:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775768187.569892:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775768187.569895:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000400:00000001:2.0:1775768187.569896:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:1.0:1775768187.569897:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000001:2.0:1775768187.569898:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000040:1.0:1775768187.569901:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000400:00000010:2.0:1775768187.569901:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768187.569904:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768187.569905:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768187.569907:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768187.569909:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207888e00 x1862022294284800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1775768187.569913:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000200:2.0:1775768187.569918:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207888e00 x1862022294284800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1775768187.569919:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:1.0:1775768187.569926:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768187.569927:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768187.569930:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775768187.569931:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768187.569934:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768187.569934:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768187.569936:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768187.569937:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775768187.569940:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00100000:2.0:1775768187.569941:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207888e00 x1862022294284800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768187.569945:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775768187.569950:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768187.569952:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1775768187.569953:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775768187.569957:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1775768187.569958:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207888e00 x1862022294284800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768212 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1775768187.569961:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768187.569965:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775768187.569969:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000100:00000001:2.0:1775768187.569971:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768187.569975:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768187.569976:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000008:00000001:1.0:1775768187.569978:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768187.569978:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768187.569980:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768187.569982:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768187.569983:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000010:1.0:1775768187.569986:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff99420788b100. 00000100:00080000:2.0:1775768187.569987:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000020:00000040:1.0:1775768187.569990:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00000100:00080000:2.0:1775768187.569994:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 02000000:00000001:1.0:1775768187.569995:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768187.569998:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768187.569998:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:1.0:1775768187.570001:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775768187.570003:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:2.0:1775768187.570005:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207888e00 x1862022294284800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768212 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1775768187.570006:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775768187.570012:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543f000. 00000100:00000001:1.0:1775768187.570018:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768187.570020:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207888e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294284800:192.168.202.156@tcp:250:kworker.0 00000100:00100000:1.0:1775768187.570025:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff99420788b100 x1862022294285184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775768187.570027:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768187.570031:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768187.570036:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207888e00 x1862022294284800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768212 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768187.570039:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775768187.570043:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768187.570044:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768187.570046:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768187.570048:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f000. 00000100:00000040:1.0:1775768187.570051:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e26cf977] to pc [ptlrpcd_rcv+-1] req@ffff99420788b100 x1862022294285184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1775768187.570052:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768187.570054:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768187.570056:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:2.0:1775768187.570059:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768187.570061:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543d400. 02000000:00000001:2.0:1775768187.570066:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:1.0:1775768187.570067:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768187.570069:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00080000:1.0:1775768187.570070:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000010:2.0:1775768187.570071:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207888e00. 00000100:00080000:1.0:1775768187.570075:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1775768187.570076:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768187.570077:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1775768187.570078:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1775768187.570092:0:36:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775768187.570093:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768187.570095:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775768187.570098:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5439 00000100:00000001:2.0:1775768187.570100:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768187.570102:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768187.570105:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768187.570105:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768187.570107:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768187.570108:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000400:00000200:1.0:1775768187.570111:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000040:2.0:1775768187.570113:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207889880 x1862022294284928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768187.570114:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775768187.570118:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768187.570122:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768187.570126:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768187.570127:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768187.570129:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768187.570130:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 02000000:00000001:2.0:1775768187.570131:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768187.570133:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1775768187.570134:0:36:0:(genops.c:811:class_conn2export()) Process entered 02000000:00000001:2.0:1775768187.570135:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1775768187.570137:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000100:00100000:2.0:1775768187.570139:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207889880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294284928:192.168.202.156@tcp:38:kworker.0 00000020:00000001:1.0:1775768187.570140:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775768187.570143:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000100:00000001:2.0:1775768187.570144:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000001:1.0:1775768187.570147:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 02000000:00000001:2.0:1775768187.570147:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768187.570148:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768187.570151:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:1.0:1775768187.570152:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 02000000:00000010:2.0:1775768187.570153:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f000. 02000000:00000001:2.0:1775768187.570178:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768187.570182:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000200:2.0:1775768187.570184:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768187.570187:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:2.0:1775768187.570188:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000100:00000040:1.0:1775768187.570191:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000010:2.0:1775768187.570191:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000001:1.0:1775768187.570196:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000200:2.0:1775768187.570196:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294284928, portal 10 00000020:00000040:1.0:1775768187.570199:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00000001:2.0:1775768187.570200:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768187.570203:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606124672 : -112966103426944 : ffff994207889880) 00000100:00080000:1.0:1775768187.570204:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775768187.570208:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768187.570211:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:2.0:1775768187.570212:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207889880 x1862022294284928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768212 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1775768187.570215:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768187.570218:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768187.570222:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775768187.570224:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:1.0:1775768187.570227:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768187.570227:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1775768187.570230:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768187.570232:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000100:00000010:1.0:1775768187.570235:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207889f80. 00000400:00000010:2.0:1775768187.570236:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000020:00000040:1.0:1775768187.570239:0:36:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00000100:00000200:2.0:1775768187.570240:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294284928, offset 0 02000000:00000001:1.0:1775768187.570243:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768187.570245:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768187.570247:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775768187.570248:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775768187.570250:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775768187.570254:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543dc00. 00000400:00000200:2.0:1775768187.570255:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00000001:1.0:1775768187.570259:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768187.570263:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00100000:1.0:1775768187.570267:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207889f80 x1862022294285312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1775768187.570268:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768187.570272:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768187.570276:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768187.570280:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768187.570283:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00000001:1.0:1775768187.570285:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:2.0:1775768187.570285:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225399:225399:256:4294967295] 192.168.202.156@tcp LPNI seq info [225395:225395:8:4294967295] 00000100:00000001:1.0:1775768187.570289:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768187.570291:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000040:1.0:1775768187.570296:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ed2274f4] to pc [ptlrpcd_rcv+-1] req@ffff994207889f80 x1862022294285312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1775768187.570301:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768187.570310:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00000001:1.0:1775768187.570316:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1775768187.570316:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768187.570320:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000040:1.0:1775768187.570322:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5450) 00000800:00000200:2.0:1775768187.570323:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775768187.570330:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768187.570333:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000100:2.0:1775768187.570337:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768187.570339:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768187.570342:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768187.570346:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768187.570348:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768187.570350:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768187.570352:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768187.570355:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207889880 x1862022294284928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768212 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768187.570363:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768187.570366:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207889880 x1862022294284928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768212 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768187.570378:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768187.570379:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768187.570390:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768187.570394:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:2.0:1775768187.570397:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768187.570399:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300112403 00000100:00000001:2.0:1775768187.570401:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300113725 00000100:00000001:2.0:1775768187.570403:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300112403 00000100:00000001:2.0:1775768187.570405:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300113725 00000100:00000001:2.0:1775768187.570407:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768187.570409:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768187.570411:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768187.570414:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768187/real 1775768187] req@ffff994207889880 x1862022294284928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768187.570422:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768187.570423:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768187.570425:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768187.570427:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768187.570428:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768187.570430:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768187.570433:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207889880 x1862022294284928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768187.570441:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207889880 x1862022294284928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768187.570447:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768187.570448:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768187.570450:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768187.570451:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768187.570454:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207889880 x1862022294284928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768187.570461:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768187.570465:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207889880 x1862022294284928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768212 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768187.570470:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768187.570471:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768187.570472:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768187.570473:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768187.570474:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768187.570476:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768187.570477:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768187.570479:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768187.570480:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768187.570482:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207889880 x1862022294284928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768212 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768187.570487:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207889880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294284928:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768187.570489:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768187.570491:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207888700 x1862022294285056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768187.570494:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768187.570495:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768187.570496:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768187.570498:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768187.570499:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768187.570500:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207888700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294285056:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768187.570502:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768187.570503:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768187.570505:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768187.570506:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768187.570508:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1ec00. 02000000:00000001:2.0:1775768187.570509:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768187.570511:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768187.570513:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768187.570515:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768187.570517:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294285056, portal 10 00000100:00000001:2.0:1775768187.570518:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768187.570519:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606120192 : -112966103431424 : ffff994207888700) 00000100:00000040:2.0:1775768187.570522:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207888700 x1862022294285056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768212 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768187.570526:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768187.570527:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768187.570528:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768187.570530:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294285056, offset 0 00000400:00000200:2.0:1775768187.570532:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768187.570534:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768187.570536:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768187.570539:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768187.570541:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768187.570543:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768187.570545:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768187.570548:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225400:225400:256:4294967295] 192.168.202.156@tcp LPNI seq info [225396:225396:8:4294967295] 00000400:00000200:2.0:1775768187.570551:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768187.570557:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768187.570561:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768187.570564:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768187.570568:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768187.570570:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775768187.570574:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768187.570576:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000100:2.0:1775768187.570580:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768187.570582:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768187.570585:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768187.570588:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768187.570591:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768187.570592:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768187.570594:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768187.570598:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207888700 x1862022294285056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768212 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768187.570605:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768187.570608:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207888700 x1862022294285056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768212 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768187.570614:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768187.570615:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768187.570618:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768187.570621:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:2.0:1775768187.570625:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768187.570627:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300112395 00000100:00000001:2.0:1775768187.570629:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300113725 00000100:00000001:2.0:1775768187.570631:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300112395 00000100:00000001:2.0:1775768187.570633:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300113725 00000100:00000001:2.0:1775768187.570635:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768187.570637:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768187.570639:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768187.570642:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768187/real 1775768187] req@ffff994207888700 x1862022294285056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768187.570649:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768187.570650:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768187.570652:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768187.570654:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768187.570655:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768187.570658:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768187.570661:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207888700 x1862022294285056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768187.570668:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207888700 x1862022294285056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768187.570674:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768187.570675:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768187.570677:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768187.570678:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768187.570681:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207888700 x1862022294285056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768187.570688:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768187.570691:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207888700 x1862022294285056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768212 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768187.570697:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768187.570699:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768187.570700:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768187.570701:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768187.570703:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768187.570705:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768187.570708:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768187.570711:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768187.570712:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768187.570716:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207888700 x1862022294285056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768212 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768187.570724:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207888700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294285056:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768187.570727:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768187.570731:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff99420788b100 x1862022294285184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768187.570737:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768187.570738:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768187.570740:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768187.570742:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768187.570744:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768187.570746:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff99420788b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294285184:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768187.570749:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768187.570751:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768187.570752:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768187.570754:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768187.570757:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1fc00. 02000000:00000001:2.0:1775768187.570759:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768187.570763:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768187.570765:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768187.570768:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768187.570771:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294285184, portal 4 00000100:00000001:2.0:1775768187.570773:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768187.570774:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606130944 : -112966103420672 : ffff99420788b100) 00000100:00000040:2.0:1775768187.570779:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff99420788b100 x1862022294285184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768212 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768187.570783:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768187.570784:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768187.570785:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768187.570787:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294285184, offset 0 00000400:00000200:2.0:1775768187.570789:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768187.570792:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768187.570794:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768187.570796:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768187.570797:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768187.570799:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768187.570800:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768187.570802:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225401:225401:256:4294967295] 192.168.202.156@tcp LPNI seq info [225397:225397:8:4294967295] 00000400:00000200:2.0:1775768187.570805:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768187.570810:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768187.570814:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768187.570817:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768187.570821:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768187.570823:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775768187.570827:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768187.570829:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000100:2.0:1775768187.570833:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768187.570835:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768187.570837:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768187.570840:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768187.570842:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768187.570844:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768187.570846:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768187.570849:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff99420788b100 x1862022294285184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768212 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768187.570856:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768187.570859:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff99420788b100 x1862022294285184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768212 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768187.570865:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768187.570866:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768187.570869:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768187.570872:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:2.0:1775768187.570874:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768187.570876:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300112418 00000100:00000001:2.0:1775768187.570878:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300113725 00000100:00000001:2.0:1775768187.570880:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300112418 00000100:00000001:2.0:1775768187.570882:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300113725 00000100:00000001:2.0:1775768187.570884:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768187.570886:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768187.570888:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768187.570891:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768187/real 1775768187] req@ffff99420788b100 x1862022294285184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768187.570898:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768187.570899:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768187.570901:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768187.570903:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768187.570905:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768187.570907:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768187.570909:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff99420788b100 x1862022294285184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768187.570917:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff99420788b100 x1862022294285184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768187.570922:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768187.570923:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768187.570924:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768187.570925:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768187.570927:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff99420788b100 x1862022294285184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768187.570931:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768187.570933:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff99420788b100 x1862022294285184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768212 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768187.570936:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768187.570937:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768187.570938:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768187.570939:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768187.570940:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768187.570942:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768187.570943:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768187.570945:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768187.570947:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768187.570949:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff99420788b100 x1862022294285184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768212 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768187.570953:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff99420788b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294285184:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768187.570955:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768187.570956:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768187.570958:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207889880 x1862022294284928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768212 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768187.570961:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768187.570962:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768187.570963:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f000. 02000000:00000001:2.0:1775768187.570965:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768187.570966:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768187.570967:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:2.0:1775768187.570968:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768187.570969:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543e400. 02000000:00000001:2.0:1775768187.570971:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768187.570971:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768187.570972:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207889880. 00000100:00000001:2.0:1775768187.570974:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768187.570975:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768187.570976:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768187.570977:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207888700 x1862022294285056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768212 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768187.570981:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768187.570982:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768187.570982:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1ec00. 02000000:00000001:2.0:1775768187.570984:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768187.570985:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768187.570986:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:2.0:1775768187.570987:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768187.570988:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543d000. 02000000:00000001:2.0:1775768187.570989:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768187.570990:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768187.570991:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207888700. 00000100:00000001:2.0:1775768187.570992:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768187.570993:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768187.570993:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768187.570995:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff99420788b100 x1862022294285184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768212 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768187.570998:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768187.570999:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768187.570999:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1fc00. 02000000:00000001:2.0:1775768187.571000:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768187.571001:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768187.571001:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:2.0:1775768187.571002:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768187.571003:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543f000. 02000000:00000001:2.0:1775768187.571004:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768187.571004:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768187.571005:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff99420788b100. 00000100:00000001:2.0:1775768187.571006:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768187.571007:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768187.571008:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768187.571010:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768187.571011:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768187.571012:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768187.571013:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768187.571014:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768187.571015:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207889f80 x1862022294285312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768187.571019:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768187.571020:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768187.571021:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768187.571022:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768187.571023:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768187.571024:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207889f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294285312:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768187.571027:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768187.571029:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768187.571030:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768187.571032:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768187.571035:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1fc00. 02000000:00000001:2.0:1775768187.571036:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768187.571040:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768187.571042:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768187.571045:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768187.571047:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294285312, portal 4 00000100:00000001:2.0:1775768187.571049:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768187.571050:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606126464 : -112966103425152 : ffff994207889f80) 00000100:00000040:2.0:1775768187.571055:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207889f80 x1862022294285312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768212 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768187.571062:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768187.571064:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768187.571066:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768187.571068:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294285312, offset 0 00000400:00000200:2.0:1775768187.571071:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768187.571074:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768187.571077:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768187.571091:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768187.571094:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768187.571096:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768187.571098:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768187.571101:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225402:225402:256:4294967295] 192.168.202.156@tcp LPNI seq info [225398:225398:8:4294967295] 00000400:00000200:2.0:1775768187.571105:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768187.571111:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768187.571114:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768187.571117:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768187.571121:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768187.571124:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775768187.571128:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768187.571130:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000100:2.0:1775768187.571133:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768187.571135:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768187.571137:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768187.571140:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768187.571143:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768187.571144:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768187.571147:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768187.571150:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207889f80 x1862022294285312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768212 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768187.571175:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768187.571178:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207889f80 x1862022294285312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768212 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768187.571185:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768187.571186:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768187.571190:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768187.571193:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:2.0:1775768187.571196:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768187.571198:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300112422 00000100:00000001:2.0:1775768187.571200:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300113725 00000100:00000001:2.0:1775768187.571203:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300112422 00000100:00000001:2.0:1775768187.571205:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300113725 00000100:00000001:2.0:1775768187.571206:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768187.571209:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768187.571210:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768187.571214:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768187/real 1775768187] req@ffff994207889f80 x1862022294285312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768187.571220:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768187.571221:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768187.571223:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768187.571225:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768187.571226:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768187.571228:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768187.571230:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207889f80 x1862022294285312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768187.571236:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207889f80 x1862022294285312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768187.571241:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768187.571242:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768187.571243:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768187.571244:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768187.571246:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207889f80 x1862022294285312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768187.571251:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768187.571254:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207889f80 x1862022294285312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768212 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768187.571258:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768187.571260:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768187.571262:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768187.571263:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768187.571264:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768187.571267:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768187.571269:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768187.571272:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768187.571273:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768187.571276:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207889f80 x1862022294285312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768212 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768187.571282:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207889f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294285312:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768187.571284:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768187.571286:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768187.571288:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207889f80 x1862022294285312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768212 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768187.571293:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768187.571294:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768187.571295:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1fc00. 02000000:00000001:2.0:1775768187.571297:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768187.571298:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768187.571299:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:2.0:1775768187.571301:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768187.571302:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543dc00. 02000000:00000001:2.0:1775768187.571304:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768187.571305:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768187.571306:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207889f80. 00000100:00000001:2.0:1775768187.571308:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768187.571309:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768187.571310:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768187.571313:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768187.571314:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768187.571315:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768187.571316:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768187.633257:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768187.633262:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768187.633268:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768187.633269:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768187.633272:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768187.633273:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768187.633276:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768187.633277:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768187.633280:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768187.633281:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768187.633324:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768187.633328:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768187.633330:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768187.697233:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768187.697239:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768187.697241:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768187.697246:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768187.697250:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768187.697253:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768187.697256:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768187.697258:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768187.697261:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768188.593197:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768188.593203:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768188.593204:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768188.593208:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768188.593210:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768188.593211:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768188.593214:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768188.593215:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768188.593216:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768188.593217:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768188.593218:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768188.593219:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768188.593220:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768188.593222:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775768188.593223:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:0.0:1775768188.593224:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768188.593225:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768188.593229:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768188.593229:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768188.593233:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768188.593233:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768188.593238:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768188.593240:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768188.593243:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768188.593243:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768188.593246:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768188.593248:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768188.657255:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768188.657261:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768188.657267:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768188.657269:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768188.657273:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768188.657274:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768188.657277:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768188.657278:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768188.657281:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768188.657282:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768188.657328:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768188.657331:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768188.657334:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768188.721247:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768188.721252:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768188.721254:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768188.721257:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768188.721260:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768188.721263:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768188.721264:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768188.721266:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768188.721268:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768188.913336:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4300113725, at 4300113728 00000100:00000010:1.0:1775768188.913352:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d6840. 00000100:00000040:0.0:1775768188.917004:0:15036:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4300113725, at 4300113731 00000100:00000010:0.0:1775768188.917017:0:15036:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d61e0. 00000100:00000001:0.0:1775768189.617255:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768189.617255:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768189.617256:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768189.617261:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768189.617262:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768189.617262:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768189.617263:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768189.617265:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768189.617265:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768189.617268:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:1.0:1775768189.617270:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775768189.617270:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768189.617271:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768189.617276:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768189.617276:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768189.617277:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768189.617280:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768189.617281:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768189.617281:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768189.617285:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768189.617289:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768189.617292:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768189.617295:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768189.617297:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768189.617298:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768189.617300:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768189.617303:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768189.681234:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768189.681240:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768189.681250:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768189.681252:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768189.681255:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768189.681257:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768189.681259:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768189.681261:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768189.681264:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768189.681265:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768189.681310:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768189.681313:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768189.681316:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768189.745236:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768189.745242:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768189.745244:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768189.745249:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768189.745253:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768189.745257:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768189.745259:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768189.745261:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768189.745264:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768190.641238:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768190.641238:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768190.641238:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768190.641244:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768190.641245:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768190.641245:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768190.641247:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768190.641248:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768190.641248:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768190.641254:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:1.0:1775768190.641254:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775768190.641255:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768190.641259:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768190.641259:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768190.641263:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768190.641264:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768190.641264:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768190.641267:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768190.641268:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768190.641271:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768190.641273:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768190.641274:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768190.641277:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768190.641279:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768190.641280:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768190.641281:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768190.641283:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768190.705264:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768190.705268:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768190.705272:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768190.705273:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768190.705275:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768190.705276:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768190.705278:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768190.705279:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768190.705281:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768190.705282:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768190.705333:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768190.705337:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768190.705340:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768190.769264:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768190.769270:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768190.769272:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768190.769277:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768190.769280:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768190.769284:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768190.769286:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768190.769288:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768190.769291:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768191.665254:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768191.665254:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768191.665254:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768191.665260:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768191.665260:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768191.665261:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768191.665262:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768191.665264:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768191.665264:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768191.665268:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:0.0:1775768191.665269:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775768191.665270:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768191.665271:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768191.665275:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768191.665276:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768191.665276:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768191.665279:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768191.665280:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768191.665280:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768191.665283:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768191.665286:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768191.665286:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768191.665289:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768191.665292:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768191.665292:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768191.665295:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768191.665298:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768191.729228:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768191.729233:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768191.729235:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768191.729238:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768191.729241:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768191.729243:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768191.729245:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768191.729247:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768191.729249:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768191.729250:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768191.729252:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768191.729253:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768191.729255:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768191.729256:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768191.729258:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768191.729258:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768191.729294:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768191.729298:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768191.729302:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768191.793261:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768191.793268:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768191.793270:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768191.793275:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768191.793279:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768191.793283:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768191.793285:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768191.793288:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768191.793290:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768192.689313:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768192.689313:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768192.689324:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768192.689324:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768192.689328:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768192.689328:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768192.689336:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775768192.689336:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768192.689344:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768192.689344:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768192.689350:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768192.689350:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1775768192.689354:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775768192.689356:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768192.689361:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1775768192.689363:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775768192.689367:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768192.689367:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768192.689368:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1775768192.689370:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1775768192.689374:0:36:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775768192.689374:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768192.689382:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775768192.689393:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5445 00000100:00000001:1.0:1775768192.689399:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768192.689404:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768192.689407:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768192.689409:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00100000:1.0:1775768192.689410:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768192.689414:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768192.689417:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768192.689421:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000400:00000200:2.0:1775768192.689421:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:1.0:1775768192.689424:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768192.689427:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768192.689428:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775768192.689433:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768192.689437:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768192.689441:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768192.689444:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775768192.689450:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775768192.689452:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:2.0:1775768192.689456:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775768192.689459:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:2.0:1775768192.689462:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:2.0:1775768192.689465:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:2.0:1775768192.689469:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768192.689471:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768192.689474:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768192.689476:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775768192.689479:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:2.0:1775768192.689484:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775768192.689487:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768192.689491:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775768192.689495:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768192.689498:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768192.689501:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775768192.689504:0:36:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775768192.689512:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfdc00. 00000020:00000040:2.0:1775768192.689516:0:36:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:2.0:1775768192.689521:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775768192.689523:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768192.689527:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775768192.689531:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775768192.689537:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775768192.689546:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1fc00. 00000100:00000001:2.0:1775768192.689555:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768192.689570:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfdc00 x1862022294285440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775768192.689581:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775768192.689589:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768192.689596:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003e25603c] to pc [ptlrpcd_rcv+-1] req@ffff994207dfdc00 x1862022294285440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768192.689621:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768192.689626:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768192.689630:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775768192.689632:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775768192.689634:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775768192.689637:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775768192.689640:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5445 00000400:00000200:2.0:1775768192.689645:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775768192.689647:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:2.0:1775768192.689650:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:3.0:1775768192.689652:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1775768192.689654:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1775768192.689656:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768192.689660:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768192.689663:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775768192.689667:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfdc00 x1862022294285440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775768192.689670:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768192.689675:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775768192.689680:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775768192.689682:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:2.0:1775768192.689684:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775768192.689686:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:2.0:1775768192.689689:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:2.0:1775768192.689691:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:2.0:1775768192.689694:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768192.689697:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768192.689699:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768192.689700:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775768192.689703:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00080000:2.0:1775768192.689705:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775768192.689710:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768192.689711:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775768192.689718:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768192.689721:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768192.689723:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775768192.689728:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775768192.689731:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768192.689734:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775768192.689738:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dff100. 00000020:00000040:2.0:1775768192.689741:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000100:00000001:3.0:1775768192.689746:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 02000000:00000001:2.0:1775768192.689747:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775768192.689750:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768192.689750:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768192.689755:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775768192.689756:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775768192.689757:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768192.689758:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768192.689762:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000010:2.0:1775768192.689764:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1ec00. 02000000:00000001:3.0:1775768192.689766:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768192.689770:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768192.689772:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294285440:192.168.202.156@tcp:250:kworker.0 00000100:00100000:2.0:1775768192.689779:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dff100 x1862022294285568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775768192.689780:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775768192.689786:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775768192.689789:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768192.689792:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1775768192.689794:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1775768192.689796:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775768192.689801:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b2c00. 00000100:00000040:2.0:1775768192.689802:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3e64ae1] to pc [ptlrpcd_rcv+-1] req@ffff994207dff100 x1862022294285568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768192.689811:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768192.689814:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768192.689817:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775768192.689819:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775768192.689820:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775768192.689823:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775768192.689826:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5445 00000400:00000200:2.0:1775768192.689830:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768192.689833:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775768192.689835:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775768192.689839:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768192.689841:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768192.689843:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768192.689845:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775768192.689847:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775768192.689848:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:2.0:1775768192.689850:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775768192.689852:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:2.0:1775768192.689854:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:2.0:1775768192.689856:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:2.0:1775768192.689859:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768192.689861:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768192.689863:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768192.689865:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775768192.689867:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:2.0:1775768192.689870:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775768192.689872:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768192.689874:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775768192.689877:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768192.689880:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768192.689881:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775768192.689884:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775768192.689887:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768192.689888:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775768192.689893:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfc700. 00000020:00000040:2.0:1775768192.689896:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:2.0:1775768192.689899:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775768192.689901:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768192.689902:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775768192.689905:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775768192.689907:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775768192.689911:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1f000. 00000100:00000001:2.0:1775768192.689925:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768192.689933:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768192.689934:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfc700 x1862022294285696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:3.0:1775768192.689946:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768192.689947:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775768192.689951:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1775768192.689952:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000100:00000040:2.0:1775768192.689957:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a176dd1] to pc [ptlrpcd_rcv+-1] req@ffff994207dfc700 x1862022294285696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768192.689966:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768192.689969:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768192.689973:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1775768192.689975:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775768192.689977:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775768192.689980:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775768192.689983:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5445 00000400:00000200:2.0:1775768192.689988:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768192.689993:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775768192.689995:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000400:00000010:3.0:1775768192.689998:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000040:2.0:1775768192.689999:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768192.690003:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768192.690006:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768192.690009:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000200:3.0:1775768192.690009:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294285440, portal 25 00000020:00000001:2.0:1775768192.690015:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:3.0:1775768192.690016:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000020:00000040:2.0:1775768192.690018:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000100:00000001:3.0:1775768192.690019:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611843584 : -112966097708032 : ffff994207dfdc00) 00000020:00000001:2.0:1775768192.690022:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775768192.690026:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:2.0:1775768192.690044:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:3.0:1775768192.690045:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfdc00 x1862022294285440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768217 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1775768192.690048:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:2.0:1775768192.690057:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768192.690062:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1775768192.690063:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768192.690067:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000040:3.0:1775768192.690071:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000100:00000001:2.0:1775768192.690072:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000010:3.0:1775768192.690076:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000020:00000040:2.0:1775768192.690077:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00000200:3.0:1775768192.690082:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294285440, offset 0 00000100:00080000:2.0:1775768192.690083:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775768192.690088:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768192.690091:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:3.0:1775768192.690094:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 02000000:00000001:2.0:1775768192.690095:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768192.690099:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768192.690101:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775768192.690104:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000400:00000200:3.0:1775768192.690107:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00000001:2.0:1775768192.690109:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768192.690112:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775768192.690116:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00000010:2.0:1775768192.690117:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfe300. 00000020:00000040:2.0:1775768192.690121:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00000400:00000200:3.0:1775768192.690124:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:2.0:1775768192.690126:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775768192.690128:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775768192.690131:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000100:00000001:2.0:1775768192.690132:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775768192.690135:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775768192.690138:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775768192.690140:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 02000000:00000010:2.0:1775768192.690143:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1f400. 00000400:00000200:3.0:1775768192.690144:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00000001:2.0:1775768192.690150:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775768192.690173:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225403:225403:256:4294967295] 192.168.202.156@tcp LPNI seq info [225399:225399:8:4294967295] 00000400:00000200:3.0:1775768192.690180:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00100000:2.0:1775768192.690187:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfe300 x1862022294285824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:3.0:1775768192.690200:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000100:00000001:2.0:1775768192.690206:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775768192.690211:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1775768192.690212:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00000040:2.0:1775768192.690219:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c661fef] to pc [ptlrpcd_rcv+-1] req@ffff994207dfe300 x1862022294285824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000010:3.0:1775768192.690219:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000100:00000001:2.0:1775768192.690230:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1775768192.690231:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00080000:2.0:1775768192.690236:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:3.0:1775768192.690238:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000100:00080000:2.0:1775768192.690246:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000200:3.1:1775768192.690249:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:2.0:1775768192.690250:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775768192.690253:0:36:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:3.1:1775768192.690254:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000100:00000001:2.0:1775768192.690260:0:36:0:(import.c:484:import_select_connection()) Process entered 00000800:00000100:3.0:1775768192.690264:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000100:00080000:2.0:1775768192.690267:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5445 00000800:00000010:3.0:1775768192.690268:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:2.0:1775768192.690274:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775768192.690275:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768192.690280:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000400:00000200:3.0:1775768192.690284:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000100:00080000:2.0:1775768192.690285:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775768192.690292:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000200:3.0:1775768192.690293:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000100:00000001:2.0:1775768192.690296:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:3.0:1775768192.690297:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000040:2.0:1775768192.690301:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768192.690303:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1775768192.690304:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775768192.690309:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000200:3.0:1775768192.690311:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfdc00 x1862022294285440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768217 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1775768192.690312:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:2.0:1775768192.690317:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775768192.690320:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:2.0:1775768192.690326:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000001:3.0:1775768192.690329:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000020:00000001:2.0:1775768192.690330:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:3.0:1775768192.690333:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfdc00 x1862022294285440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768217 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775768192.690337:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768192.690342:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768192.690347:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768192.690349:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768192.690353:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768192.690354:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775768192.690358:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00000001:3.0:1775768192.690359:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768192.690363:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775768192.690368:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775768192.690368:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:2.0:1775768192.690373:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1775768192.690373:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4300118846 02000000:00000001:2.0:1775768192.690377:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000010:3.0:1775768192.690379:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207981360. 02000000:00000001:2.0:1775768192.690380:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768192.690385:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775768192.690387:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000100:00000001:3.0:1775768192.690390:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000008:00000001:2.0:1775768192.690391:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768192.690395:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768192.690398:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4300118846 00000100:00000010:2.0:1775768192.690399:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfed80. 00000020:00000040:2.0:1775768192.690404:0:36:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00000100:00000010:3.0:1775768192.690404:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207981420. 02000000:00000001:2.0:1775768192.690408:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1775768192.690410:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 02000000:00000001:2.0:1775768192.690411:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768192.690415:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1775768192.690416:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768192.690417:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775768192.690421:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768192.690422:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768192.690426:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 02000000:00000010:2.0:1775768192.690428:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1d000. 00000100:00100000:3.0:1775768192.690432:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768192/real 1775768192] req@ffff994207dfdc00 x1862022294285440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768192.690434:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768192.690444:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfed80 x1862022294285952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:3.0:1775768192.690447:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775768192.690450:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775768192.690454:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000100:00000001:2.0:1775768192.690459:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:3.0:1775768192.690460:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775768192.690464:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:2.0:1775768192.690465:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768192.690469:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:2.0:1775768192.690473:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb9b9eae] to pc [ptlrpcd_rcv+-1] req@ffff994207dfed80 x1862022294285952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775768192.690476:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfdc00 x1862022294285440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768192.690495:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768192.690501:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5455) 00000100:00000200:3.0:1775768192.690509:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfdc00 x1862022294285440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768192.690516:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775768192.690518:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768192.690520:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775768192.690522:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775768192.690526:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfdc00 x1862022294285440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768192.690533:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775768192.690537:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfdc00 x1862022294285440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768217 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768192.690544:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775768192.690546:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768192.690548:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775768192.690549:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768192.690552:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775768192.690554:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775768192.690557:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768192.690561:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775768192.690564:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775768192.690568:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfdc00 x1862022294285440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768217 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768192.690576:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294285440:192.168.202.156@tcp:250:kworker.0 00000100:00000001:3.0:1775768192.690581:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768192.690583:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775768192.690586:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfdc00 x1862022294285440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768217 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768192.690597:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775768192.690600:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775768192.690602:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b2c00. 02000000:00000001:3.0:1775768192.690605:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775768192.690606:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775768192.690608:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:3.0:1775768192.690610:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775768192.690613:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1fc00. 02000000:00000001:3.0:1775768192.690617:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775768192.690619:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775768192.690620:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfdc00. 00000100:00000001:3.0:1775768192.690624:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775768192.690625:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768192.690628:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768192.690635:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768192.690637:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768192.690639:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775768192.690640:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768192.690642:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775768192.690645:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dff100 x1862022294285568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768192.690651:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768192.690653:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768192.690654:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775768192.690656:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775768192.690658:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768192.690660:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294285568:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775768192.690664:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775768192.690666:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775768192.690667:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768192.690669:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775768192.690672:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b2c00. 02000000:00000001:3.0:1775768192.690673:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775768192.690677:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775768192.690680:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775768192.690682:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775768192.690685:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294285568, portal 10 00000100:00000001:3.0:1775768192.690687:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775768192.690688:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611848960 : -112966097702656 : ffff994207dff100) 00000100:00000040:3.0:1775768192.690695:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dff100 x1862022294285568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768217 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768192.690702:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775768192.690704:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775768192.690706:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775768192.690709:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294285568, offset 0 00000400:00000200:3.0:1775768192.690712:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775768192.690716:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775768192.690720:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775768192.690723:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775768192.690725:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775768192.690727:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775768192.690729:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775768192.690732:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225404:225404:256:4294967295] 192.168.202.156@tcp LPNI seq info [225400:225400:8:4294967295] 00000400:00000200:3.0:1775768192.690736:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775768192.690742:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775768192.690746:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775768192.690749:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775768192.690753:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775768192.690755:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775768192.690759:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775768192.690761:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000100:3.0:1775768192.690765:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775768192.690766:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775768192.690769:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775768192.690772:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775768192.690775:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775768192.690776:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775768192.690779:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775768192.690782:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dff100 x1862022294285568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768217 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768192.690788:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775768192.690791:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dff100 x1862022294285568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768217 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768192.690797:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768192.690799:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775768192.690801:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768192.690805:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:3.0:1775768192.690808:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775768192.690811:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300117523 00000100:00000001:3.0:1775768192.690813:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300118846 00000100:00000001:3.0:1775768192.690816:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300117523 00000100:00000001:3.0:1775768192.690817:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300118846 00000100:00000001:3.0:1775768192.690820:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775768192.690821:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768192.690822:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775768192.690826:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768192/real 1775768192] req@ffff994207dff100 x1862022294285568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775768192.690833:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775768192.690834:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775768192.690836:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775768192.690838:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775768192.690839:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775768192.690841:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775768192.690843:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dff100 x1862022294285568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775768192.690851:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dff100 x1862022294285568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768192.690861:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775768192.690863:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768192.690864:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775768192.690866:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775768192.690868:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dff100 x1862022294285568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768192.690874:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775768192.690877:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dff100 x1862022294285568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768217 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768192.690882:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775768192.690885:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768192.690886:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775768192.690887:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768192.690889:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775768192.690892:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775768192.690893:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768192.690896:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775768192.690898:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775768192.690901:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dff100 x1862022294285568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768217 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768192.690908:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294285568:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775768192.690912:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775768192.690916:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfc700 x1862022294285696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768192.690921:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768192.690922:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768192.690925:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775768192.690926:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775768192.690928:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768192.690931:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294285696:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775768192.690934:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775768192.690936:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775768192.690937:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768192.690939:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775768192.690942:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b2800. 02000000:00000001:3.0:1775768192.690944:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775768192.690947:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775768192.690949:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775768192.690951:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775768192.690954:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294285696, portal 10 00000100:00000001:3.0:1775768192.690956:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775768192.690957:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611838208 : -112966097713408 : ffff994207dfc700) 00000100:00000040:3.0:1775768192.690962:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfc700 x1862022294285696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768217 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768192.690967:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775768192.690969:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775768192.690971:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775768192.690973:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294285696, offset 0 00000400:00000200:3.0:1775768192.690976:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775768192.690980:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775768192.690982:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775768192.690985:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775768192.690987:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775768192.690989:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775768192.690992:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775768192.690995:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225405:225405:256:4294967295] 192.168.202.156@tcp LPNI seq info [225401:225401:8:4294967295] 00000400:00000200:3.0:1775768192.690998:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775768192.691004:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775768192.691008:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775768192.691011:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775768192.691014:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775768192.691017:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775768192.691021:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775768192.691023:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000100:3.0:1775768192.691027:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775768192.691041:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775768192.691044:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775768192.691047:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775768192.691050:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775768192.691051:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775768192.691054:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775768192.691056:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfc700 x1862022294285696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768217 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768192.691064:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775768192.691066:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfc700 x1862022294285696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768217 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768192.691071:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768192.691073:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775768192.691076:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768192.691079:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:3.0:1775768192.691083:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775768192.691084:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300117515 00000100:00000001:3.0:1775768192.691087:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300118846 00000100:00000001:3.0:1775768192.691089:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300117515 00000100:00000001:3.0:1775768192.691091:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300118846 00000100:00000001:3.0:1775768192.691093:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775768192.691095:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768192.691096:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775768192.691101:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768192/real 1775768192] req@ffff994207dfc700 x1862022294285696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775768192.691109:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775768192.691110:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775768192.691111:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775768192.691114:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775768192.691115:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775768192.691117:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775768192.691120:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfc700 x1862022294285696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775768192.691128:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfc700 x1862022294285696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768192.691134:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775768192.691135:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768192.691137:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775768192.691138:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775768192.691141:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfc700 x1862022294285696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768192.691147:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775768192.691150:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfc700 x1862022294285696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768217 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768192.691177:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775768192.691181:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768192.691183:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775768192.691184:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768192.691186:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775768192.691189:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775768192.691192:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768192.691195:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775768192.691198:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775768192.691203:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfc700 x1862022294285696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768217 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768192.691211:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294285696:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775768192.691214:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775768192.691218:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfe300 x1862022294285824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768192.691223:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768192.691224:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768192.691227:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775768192.691228:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775768192.691230:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768192.691233:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294285824:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775768192.691236:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775768192.691238:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775768192.691240:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768192.691242:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775768192.691245:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b0000. 02000000:00000001:3.0:1775768192.691247:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775768192.691250:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775768192.691253:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775768192.691256:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775768192.691259:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294285824, portal 4 00000100:00000001:3.0:1775768192.691260:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775768192.691262:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611845376 : -112966097706240 : ffff994207dfe300) 00000100:00000040:3.0:1775768192.691267:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfe300 x1862022294285824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768217 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768192.691273:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775768192.691275:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775768192.691278:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775768192.691280:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294285824, offset 0 00000400:00000200:3.0:1775768192.691285:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775768192.691288:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775768192.691291:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775768192.691294:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775768192.691297:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775768192.691299:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775768192.691301:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775768192.691304:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225406:225406:256:4294967295] 192.168.202.156@tcp LPNI seq info [225402:225402:8:4294967295] 00000400:00000200:3.0:1775768192.691308:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775768192.691313:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775768192.691318:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775768192.691321:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775768192.691324:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775768192.691326:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775768192.691330:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775768192.691333:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000100:3.0:1775768192.691336:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775768192.691338:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775768192.691341:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775768192.691344:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775768192.691347:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775768192.691348:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775768192.691350:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775768192.691353:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfe300 x1862022294285824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768217 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768192.691361:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775768192.691363:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfe300 x1862022294285824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768217 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768192.691369:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768192.691371:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775768192.691373:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768192.691376:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:3.0:1775768192.691379:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775768192.691381:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300117539 00000100:00000001:3.0:1775768192.691384:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300118846 00000100:00000001:3.0:1775768192.691386:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300117539 00000100:00000001:3.0:1775768192.691388:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300118846 00000100:00000001:3.0:1775768192.691390:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775768192.691392:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768192.691393:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775768192.691396:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768192/real 1775768192] req@ffff994207dfe300 x1862022294285824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775768192.691404:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775768192.691405:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775768192.691407:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775768192.691409:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775768192.691410:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775768192.691412:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775768192.691415:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfe300 x1862022294285824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775768192.691423:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfe300 x1862022294285824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768192.691429:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775768192.691430:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768192.691432:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775768192.691433:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775768192.691436:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfe300 x1862022294285824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768192.691443:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775768192.691446:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfe300 x1862022294285824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768217 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768192.691453:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775768192.691456:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768192.691458:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775768192.691459:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768192.691460:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775768192.691464:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775768192.691467:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768192.691469:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775768192.691471:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775768192.691475:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfe300 x1862022294285824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768217 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768192.691483:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294285824:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775768192.691486:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775768192.691489:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfed80 x1862022294285952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768192.691496:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768192.691497:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768192.691499:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775768192.691500:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775768192.691502:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768192.691505:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294285952:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775768192.691508:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775768192.691509:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775768192.691511:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768192.691513:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775768192.691516:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b0c00. 02000000:00000001:3.0:1775768192.691518:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775768192.691521:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775768192.691524:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775768192.691526:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775768192.691529:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294285952, portal 4 00000100:00000001:3.0:1775768192.691531:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775768192.691533:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611848064 : -112966097703552 : ffff994207dfed80) 00000100:00000040:3.0:1775768192.691537:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfed80 x1862022294285952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768217 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768192.691544:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775768192.691545:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775768192.691547:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775768192.691549:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294285952, offset 0 00000400:00000200:3.0:1775768192.691552:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775768192.691556:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775768192.691559:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775768192.691562:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775768192.691564:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775768192.691566:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775768192.691568:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775768192.691571:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225407:225407:256:4294967295] 192.168.202.156@tcp LPNI seq info [225403:225403:8:4294967295] 00000400:00000200:3.0:1775768192.691575:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775768192.691580:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775768192.691584:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775768192.691588:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775768192.691591:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775768192.691593:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775768192.691597:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775768192.691599:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 37 secs later) 00000800:00000100:3.0:1775768192.691603:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775768192.691605:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775768192.691607:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775768192.691610:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775768192.691613:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775768192.691614:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775768192.691616:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775768192.691619:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfed80 x1862022294285952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768217 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768192.691631:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775768192.691634:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfed80 x1862022294285952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768217 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768192.691641:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768192.691642:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775768192.691644:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768192.691646:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:3.0:1775768192.691649:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775768192.691650:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300117542 00000100:00000001:3.0:1775768192.691653:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300118846 00000100:00000001:3.0:1775768192.691655:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300117542 00000100:00000001:3.0:1775768192.691657:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300118846 00000100:00000001:3.0:1775768192.691659:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775768192.691661:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768192.691662:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775768192.691665:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768192/real 1775768192] req@ffff994207dfed80 x1862022294285952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775768192.691672:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775768192.691673:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775768192.691674:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775768192.691676:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775768192.691678:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775768192.691680:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775768192.691684:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfed80 x1862022294285952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775768192.691691:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfed80 x1862022294285952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768192.691697:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775768192.691699:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768192.691701:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775768192.691702:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775768192.691705:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfed80 x1862022294285952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768192.691711:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775768192.691714:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfed80 x1862022294285952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768217 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768192.691720:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775768192.691722:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768192.691723:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775768192.691724:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768192.691726:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775768192.691730:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775768192.691732:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768192.691734:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775768192.691736:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775768192.691740:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfed80 x1862022294285952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768217 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768192.691747:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294285952:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775768192.691750:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768192.691752:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775768192.691755:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dff100 x1862022294285568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768217 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768192.691761:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775768192.691762:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775768192.691764:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b2c00. 02000000:00000001:3.0:1775768192.691766:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775768192.691767:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775768192.691769:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:3.0:1775768192.691771:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775768192.691773:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1ec00. 02000000:00000001:3.0:1775768192.691776:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775768192.691777:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775768192.691779:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dff100. 00000100:00000001:3.0:1775768192.691781:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775768192.691783:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768192.691784:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775768192.691788:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfc700 x1862022294285696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768217 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768192.691795:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775768192.691796:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775768192.691797:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b2800. 02000000:00000001:3.0:1775768192.691799:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775768192.691800:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775768192.691800:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:3.0:1775768192.691802:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775768192.691803:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1f000. 02000000:00000001:3.0:1775768192.691804:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775768192.691805:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775768192.691806:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfc700. 00000100:00000001:3.0:1775768192.691808:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775768192.691809:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768192.691810:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775768192.691812:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfe300 x1862022294285824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768217 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768192.691817:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775768192.691818:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775768192.691819:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b0000. 02000000:00000001:3.0:1775768192.691822:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775768192.691823:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775768192.691824:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:3.0:1775768192.691826:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775768192.691827:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1f400. 02000000:00000001:3.0:1775768192.691828:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775768192.691830:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775768192.691831:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfe300. 00000100:00000001:3.0:1775768192.691833:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775768192.691834:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768192.691836:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775768192.691839:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfed80 x1862022294285952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768217 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768192.691844:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775768192.691844:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775768192.691846:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b0c00. 02000000:00000001:3.0:1775768192.691847:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775768192.691848:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775768192.691850:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:3.0:1775768192.691851:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775768192.691853:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1d000. 02000000:00000001:3.0:1775768192.691856:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775768192.691857:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775768192.691858:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfed80. 00000100:00000001:3.0:1775768192.691860:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775768192.691862:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768192.691863:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768192.691866:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768192.691868:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768192.691870:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775768192.691871:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768192.753285:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768192.753293:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768192.753301:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768192.753303:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768192.753307:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768192.753310:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768192.753314:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768192.753315:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768192.753319:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768192.753321:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768192.753384:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768192.753389:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768192.753392:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768192.817296:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768192.817303:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768192.817305:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768192.817311:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768192.817314:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768192.817318:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768192.817321:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768192.817323:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768192.817325:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768193.713273:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768193.713274:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768193.713280:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768193.713280:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768193.713283:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768193.713284:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768193.713287:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:1.0:1775768193.713291:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775768193.713291:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768193.713292:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768193.713295:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768193.713298:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768193.713298:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:2.0:1775768193.713299:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768193.713303:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768193.713304:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768193.713304:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768193.713308:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768193.713309:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768193.713312:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768193.713316:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768193.713316:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768193.713319:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768193.713322:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768193.713323:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768193.713326:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768193.713329:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768193.777236:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768193.777241:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768193.777247:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768193.777248:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768193.777252:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768193.777253:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768193.777256:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768193.777257:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768193.777259:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768193.777260:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768193.777313:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768193.777317:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768193.777320:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768193.841278:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768193.841284:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768193.841286:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768193.841292:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768193.841296:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768193.841300:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768193.841302:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768193.841304:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768193.841307:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768194.033234:0:15036:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4300118846, at 4300118848 00000100:00000040:1.0:1775768194.033234:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4300118846, at 4300118848 00000100:00000010:1.0:1775768194.033246:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207981420. 00000100:00000010:0.0:1775768194.033248:0:15036:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207981360. 00000100:00000001:2.0:1775768194.737283:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768194.737283:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768194.737293:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768194.737293:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768194.737294:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768194.737297:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768194.737298:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768194.737299:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768194.737304:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768194.737308:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:1.0:1775768194.737309:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775768194.737309:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768194.737326:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768194.737326:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768194.737327:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768194.737338:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768194.737338:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768194.737340:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768194.737348:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768194.737352:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768194.737355:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768194.737357:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768194.737360:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768194.737363:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768194.737364:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768194.737370:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768194.737375:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768194.801282:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768194.801286:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768194.801291:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768194.801292:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768194.801295:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768194.801296:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768194.801298:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768194.801299:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768194.801304:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768194.801304:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768194.801355:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768194.801359:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768194.801363:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768194.865258:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768194.865265:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768194.865267:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768194.865272:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768194.865276:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768194.865279:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768194.865282:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768194.865284:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768194.865287:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768195.761248:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768195.761248:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768195.761248:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768195.761253:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768195.761253:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768195.761254:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768195.761257:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768195.761258:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:3.0:1775768195.761259:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768195.761261:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768195.761261:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768195.761263:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1775768195.761266:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768195.761267:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768195.761267:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768195.761271:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768195.761271:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768195.761271:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768195.761274:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768195.761274:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768195.761279:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768195.761281:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768195.761284:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768195.761286:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768195.761287:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768195.761289:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768195.761292:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768195.825244:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768195.825250:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768195.825256:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768195.825258:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768195.825261:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768195.825262:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768195.825266:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768195.825267:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768195.825270:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768195.825271:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768195.825329:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768195.825333:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768195.825336:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768195.889255:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768195.889261:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768195.889263:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768195.889268:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768195.889271:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768195.889275:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768195.889278:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768195.889280:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768195.889282:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768196.785237:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768196.785237:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768196.785237:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768196.785243:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768196.785243:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768196.785244:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768196.785245:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768196.785247:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768196.785247:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768196.785249:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:1.0:1775768196.785253:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775768196.785253:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768196.785254:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768196.785258:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768196.785258:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768196.785259:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768196.785261:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768196.785263:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768196.785263:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768196.785264:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768196.785269:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768196.785269:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768196.785272:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768196.785275:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768196.785276:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768196.785278:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768196.785281:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768196.849222:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768196.849226:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768196.849230:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768196.849233:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768196.849235:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768196.849237:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768196.849239:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768196.849240:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768196.849241:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768196.849243:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768196.849244:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768196.849245:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768196.849246:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768196.849247:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768196.849248:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768196.849249:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768196.849250:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768196.849251:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768196.849252:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768196.849253:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768196.849254:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768196.849255:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768196.849256:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768196.849256:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768196.849257:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768196.849258:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768196.849260:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768196.849261:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768196.849262:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768196.849262:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768196.849263:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768196.849264:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768196.849265:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768196.849266:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768196.849307:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768196.849311:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768196.849314:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768196.913280:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768196.913286:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768196.913288:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768196.913293:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768196.913297:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768196.913301:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768196.913303:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768196.913306:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768196.913309:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768197.809267:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768197.809273:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768197.809275:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768197.809277:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775768197.809280:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:3.0:1775768197.809282:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00080000:2.0:1775768197.809283:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775768197.809285:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768197.809288:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768197.809288:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1775768197.809289:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1775768197.809293:0:36:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775768197.809294:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768197.809294:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768197.809296:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768197.809300:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768197.809300:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768197.809301:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768197.809302:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:0.0:1775768197.809303:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768197.809304:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768197.809305:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768197.809307:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00080000:2.0:1775768197.809310:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5450 00000100:00000001:0.0:1775768197.809312:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768197.809312:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768197.809312:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768197.809314:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768197.809316:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768197.809319:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768197.809320:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768197.809322:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1775768197.809325:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768197.809326:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768197.809343:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775768197.809347:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775768197.809351:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768197.809355:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768197.809359:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768197.809362:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775768197.809366:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775768197.809369:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:2.0:1775768197.809372:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775768197.809374:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:2.0:1775768197.809376:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:2.0:1775768197.809378:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:2.0:1775768197.809381:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768197.809383:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768197.809384:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768197.809386:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775768197.809389:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:2.0:1775768197.809392:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775768197.809395:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768197.809398:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775768197.809401:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768197.809403:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768197.809411:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775768197.809414:0:36:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775768197.809422:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dff480. 00000020:00000040:2.0:1775768197.809425:0:36:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:2.0:1775768197.809428:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775768197.809430:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768197.809433:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775768197.809435:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775768197.809438:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775768197.809445:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1e400. 00000100:00000001:2.0:1775768197.809451:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768197.809458:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dff480 x1862022294286080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775768197.809468:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775768197.809472:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768197.809476:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093a15639] to pc [ptlrpcd_rcv+-1] req@ffff994207dff480 x1862022294286080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768197.809498:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768197.809502:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768197.809504:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775768197.809505:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775768197.809507:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775768197.809509:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775768197.809511:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5450 00000400:00000200:2.0:1775768197.809515:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768197.809517:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775768197.809520:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775768197.809523:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768197.809526:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768197.809528:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768197.809529:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768197.809530:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:3.0:1775768197.809533:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1775768197.809534:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775768197.809536:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000100:00000001:3.0:1775768197.809537:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000001:2.0:1775768197.809538:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775768197.809541:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:2.0:1775768197.809544:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:3.0:1775768197.809546:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dff480 x1862022294286080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1775768197.809547:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:2.0:1775768197.809555:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768197.809560:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1775768197.809560:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768197.809564:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768197.809566:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000001:3.0:1775768197.809568:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1775768197.809569:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 02000000:00000001:3.0:1775768197.809570:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:2.0:1775768197.809571:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 02000000:00000001:3.0:1775768197.809573:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768197.809577:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00100000:3.0:1775768197.809579:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294286080:192.168.202.156@tcp:250:kworker.0 00000100:00000001:2.0:1775768197.809584:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768197.809585:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768197.809588:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775768197.809589:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775768197.809591:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768197.809593:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775768197.809593:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1775768197.809597:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775768197.809598:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b0c00. 00000100:00000001:2.0:1775768197.809599:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:3.0:1775768197.809602:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1775768197.809604:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000400:00000200:3.0:1775768197.809608:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00000001:2.0:1775768197.809609:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768197.809612:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1775768197.809612:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000100:00000010:2.0:1775768197.809618:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfce00. 00000400:00000010:3.0:1775768197.809619:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000020:00000040:2.0:1775768197.809621:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:2.0:1775768197.809624:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000200:3.0:1775768197.809625:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294286080, portal 25 02000000:00000001:2.0:1775768197.809626:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768197.809628:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768197.809629:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1775768197.809630:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611849856 : -112966097701760 : ffff994207dff480) 02000000:00000001:2.0:1775768197.809631:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775768197.809635:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768197.809638:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dff480 x1862022294286080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768222 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1775768197.809641:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1c800. 00000100:00000001:3.0:1775768197.809646:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1775768197.809647:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768197.809649:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775768197.809653:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00100000:2.0:1775768197.809656:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfce00 x1862022294286208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:3.0:1775768197.809656:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294286080, offset 0 00000400:00000200:3.0:1775768197.809664:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00000001:2.0:1775768197.809669:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775768197.809671:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775768197.809674:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000040:2.0:1775768197.809679:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e5a7debe] to pc [ptlrpcd_rcv+-1] req@ffff994207dfce00 x1862022294286208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1775768197.809680:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775768197.809686:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775768197.809691:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000100:00000001:2.0:1775768197.809696:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775768197.809696:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775768197.809700:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00080000:2.0:1775768197.809701:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:3.0:1775768197.809705:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225408:225408:256:4294967295] 192.168.202.156@tcp LPNI seq info [225404:225404:8:4294967295] 00000100:00080000:2.0:1775768197.809707:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775768197.809710:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775768197.809713:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:3.0:1775768197.809713:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768197.809718:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775768197.809721:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5450 00000400:00000200:2.0:1775768197.809728:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775768197.809731:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000400:00000200:2.0:1775768197.809735:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775768197.809738:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000800:00000200:3.0:1775768197.809742:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00000040:2.0:1775768197.809744:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768197.809748:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000010:3.0:1775768197.809749:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000100:00000040:2.0:1775768197.809750:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768197.809754:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775768197.809756:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775768197.809758:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000800:00000200:3.0:1775768197.809758:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000001:2.0:1775768197.809762:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000800:00000200:3.0:1775768197.809762:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000020:00000040:2.0:1775768197.809763:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:2.0:1775768197.809767:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000800:00000200:3.1:1775768197.809768:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000001:2.0:1775768197.809771:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000800:00000200:3.1:1775768197.809772:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000100:00000040:2.0:1775768197.809779:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000800:00000100:3.0:1775768197.809780:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000100:00000001:2.0:1775768197.809784:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768197.809787:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768197.809789:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000800:00000010:3.0:1775768197.809790:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000020:00000040:2.0:1775768197.809791:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:2.0:1775768197.809796:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775768197.809798:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775768197.809799:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000001:2.0:1775768197.809800:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775768197.809804:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1775768197.809804:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 02000000:00000001:2.0:1775768197.809809:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775768197.809811:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000100:00000001:2.0:1775768197.809812:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000010:3.0:1775768197.809813:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000008:00100000:2.0:1775768197.809815:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1775768197.809817:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000008:00000001:2.0:1775768197.809818:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768197.809821:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1775768197.809823:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dff480 x1862022294286080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768222 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1775768197.809826:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfed80. 00000020:00000040:2.0:1775768197.809831:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000100:00000001:3.0:1775768197.809833:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1775768197.809835:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775768197.809837:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768197.809837:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dff480 x1862022294286080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768222 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768197.809842:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775768197.809845:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1775768197.809847:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768197.809848:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768197.809850:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775768197.809855:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775768197.809858:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775768197.809859:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4300123965 02000000:00000010:2.0:1775768197.809861:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1d400. 00000100:00000010:3.0:1775768197.809863:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207981b40. 00000100:00000001:2.0:1775768197.809866:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768197.809870:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00100000:2.0:1775768197.809874:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfed80 x1862022294286336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775768197.809876:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4300123965 00000100:00000010:3.0:1775768197.809880:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207981360. 00000100:00000001:3.0:1775768197.809884:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1775768197.809887:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768197.809889:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775768197.809890:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768197.809893:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1775768197.809894:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768197.809897:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768197/real 1775768197] req@ffff994207dff480 x1862022294286080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775768197.809900:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb9b9eae] to pc [ptlrpcd_rcv+-1] req@ffff994207dfed80 x1862022294286336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775768197.809910:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775768197.809912:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775768197.809915:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000100:00000001:2.0:1775768197.809916:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775768197.809917:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775768197.809920:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00080000:2.0:1775768197.809921:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1775768197.809922:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00080000:2.0:1775768197.809926:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000200:3.0:1775768197.809927:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dff480 x1862022294286080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768197.809930:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775768197.809932:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:3.0:1775768197.809938:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dff480 x1862022294286080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768197.809939:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775768197.809944:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5450 00000100:00000001:3.0:1775768197.809950:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775768197.809951:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1775768197.809952:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775768197.809954:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775768197.809956:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1775768197.809959:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00100000:3.0:1775768197.809960:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dff480 x1862022294286080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768197.809964:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1775768197.809967:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768197.809971:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000040:3.0:1775768197.809973:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dff480 x1862022294286080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768222 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768197.809975:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768197.809992:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768197.809994:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775768197.809997:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775768197.809998:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:2.0:1775768197.809999:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775768197.810001:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:2.0:1775768197.810006:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:3.0:1775768197.810007:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:2.0:1775768197.810010:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:3.0:1775768197.810011:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768197.810013:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775768197.810015:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768197.810017:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768197.810018:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768197.810022:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1775768197.810022:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775768197.810026:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000040:2.0:1775768197.810028:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768197.810031:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00080000:3.0:1775768197.810031:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775768197.810034:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000040:2.0:1775768197.810035:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00000040:3.0:1775768197.810040:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dff480 x1862022294286080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768222 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768197.810042:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775768197.810048:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768197.810052:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:3.0:1775768197.810055:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294286080:192.168.202.156@tcp:250:kworker.0 02000000:00000001:2.0:1775768197.810057:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775768197.810059:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1775768197.810060:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768197.810061:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1775768197.810064:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000040:3.0:1775768197.810064:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dff480 x1862022294286080/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768222 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1775768197.810068:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:2.0:1775768197.810074:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768197.810075:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1775768197.810077:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768197.810078:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775768197.810081:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b0c00. 00000100:00000010:2.0:1775768197.810083:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfe300. 02000000:00000001:3.0:1775768197.810084:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000040:2.0:1775768197.810087:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:3.0:1775768197.810087:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775768197.810089:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 02000000:00000001:2.0:1775768197.810092:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000020:00000001:3.0:1775768197.810092:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000001:2.0:1775768197.810095:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775768197.810095:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1e400. 00000100:00000001:2.0:1775768197.810100:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775768197.810101:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775768197.810101:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775768197.810104:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:2.0:1775768197.810105:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775768197.810106:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dff480. 00000100:00000001:3.0:1775768197.810109:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 02000000:00000010:2.0:1775768197.810110:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1e800. 00000100:00000001:3.0:1775768197.810110:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768197.810115:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768197.810116:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768197.810121:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:2.0:1775768197.810123:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfe300 x1862022294286464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775768197.810124:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768197.810129:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775768197.810132:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768197.810135:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1775768197.810136:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775768197.810139:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768197.810143:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfce00 x1862022294286208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775768197.810146:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c661fef] to pc [ptlrpcd_rcv+-1] req@ffff994207dfe300 x1862022294286464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768197.810178:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768197.810179:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768197.810184:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775768197.810187:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775768197.810189:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768197.810190:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768197.810194:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775768197.810195:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294286208:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775768197.810201:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00080000:2.0:1775768197.810202:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 02000000:00000001:3.0:1775768197.810204:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1775768197.810205:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:3.0:1775768197.810206:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768197.810209:0:36:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:3.0:1775768197.810209:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775768197.810212:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b0c00. 00000100:00000001:2.0:1775768197.810213:0:36:0:(import.c:484:import_select_connection()) Process entered 02000000:00000001:3.0:1775768197.810216:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768197.810218:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5450 00000400:00000200:3.0:1775768197.810222:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775768197.810224:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000200:2.0:1775768197.810226:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775768197.810227:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000400:00000200:2.0:1775768197.810231:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000200:3.0:1775768197.810231:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294286208, portal 10 00000100:00000001:3.0:1775768197.810235:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00080000:2.0:1775768197.810236:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1775768197.810237:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611840000 : -112966097711616 : ffff994207dfce00) 00000100:00000040:2.0:1775768197.810241:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768197.810244:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775768197.810245:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfce00 x1862022294286208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768222 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775768197.810249:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768197.810254:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:3.0:1775768197.810256:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:2.0:1775768197.810257:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775768197.810260:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000100:00000040:3.0:1775768197.810260:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000020:00000001:2.0:1775768197.810263:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000010:3.0:1775768197.810264:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000020:00000040:2.0:1775768197.810266:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000100:00000200:3.0:1775768197.810267:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294286208, offset 0 00000020:00000001:2.0:1775768197.810271:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000400:00000200:3.0:1775768197.810272:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000020:00000001:2.0:1775768197.810276:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000400:00000200:3.0:1775768197.810277:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000040:2.0:1775768197.810284:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000200:3.0:1775768197.810284:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00000001:2.0:1775768197.810289:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1775768197.810289:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:2.0:1775768197.810294:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000200:3.0:1775768197.810294:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000100:00000001:2.0:1775768197.810299:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000200:3.0:1775768197.810299:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000020:00000040:2.0:1775768197.810304:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000400:00000200:3.0:1775768197.810305:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00080000:2.0:1775768197.810309:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775768197.810314:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775768197.810314:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225409:225409:256:4294967295] 192.168.202.156@tcp LPNI seq info [225405:225405:8:4294967295] 02000000:00000001:2.0:1775768197.810317:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775768197.810321:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1775768197.810322:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:2.0:1775768197.810326:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768197.810328:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775768197.810331:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:2.0:1775768197.810336:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775768197.810336:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000100:00000001:2.0:1775768197.810339:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775768197.810345:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfc700. 00000800:00000200:3.0:1775768197.810345:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000020:00000040:2.0:1775768197.810349:0:36:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00000800:00000010:3.0:1775768197.810350:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 02000000:00000001:2.0:1775768197.810352:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775768197.810354:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768197.810358:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000800:00000200:3.0:1775768197.810358:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 02000000:00000001:2.0:1775768197.810361:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:3.0:1775768197.810362:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 02000000:00000001:2.0:1775768197.810364:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1775768197.810368:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 02000000:00000010:2.0:1775768197.810370:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1e400. 00000800:00000200:3.1:1775768197.810370:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000100:00000001:2.0:1775768197.810376:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1775768197.810376:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775768197.810377:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775768197.810381:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00100000:2.0:1775768197.810385:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfc700 x1862022294286592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:3.0:1775768197.810386:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775768197.810390:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775768197.810392:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775768197.810396:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775768197.810400:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfce00 x1862022294286208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768222 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768197.810402:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775768197.810407:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768197.810412:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768197.810417:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a176dd1] to pc [ptlrpcd_rcv+-1] req@ffff994207dfc700 x1862022294286592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775768197.810417:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfce00 x1862022294286208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768222 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768197.810439:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768197.810442:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768197.810442:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775768197.810446:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768197.810448:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5460) 00000100:00000040:3.0:1775768197.810452:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:3.0:1775768197.810456:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775768197.810459:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300122643 00000100:00000001:3.0:1775768197.810462:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300123965 00000100:00000001:3.0:1775768197.810464:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300122643 00000100:00000001:3.0:1775768197.810467:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300123965 00000100:00000001:3.0:1775768197.810469:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775768197.810471:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768197.810473:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775768197.810475:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768197/real 1775768197] req@ffff994207dfce00 x1862022294286208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775768197.810481:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775768197.810481:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775768197.810483:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775768197.810484:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775768197.810485:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775768197.810486:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775768197.810488:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfce00 x1862022294286208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775768197.810494:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfce00 x1862022294286208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768197.810500:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775768197.810501:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768197.810503:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775768197.810504:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775768197.810507:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfce00 x1862022294286208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768197.810513:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775768197.810517:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfce00 x1862022294286208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768222 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768197.810522:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775768197.810524:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768197.810525:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775768197.810526:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768197.810528:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775768197.810530:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775768197.810532:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768197.810535:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775768197.810537:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775768197.810540:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfce00 x1862022294286208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768222 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768197.810546:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294286208:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775768197.810549:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775768197.810551:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfed80 x1862022294286336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768197.810557:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768197.810558:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768197.810560:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775768197.810562:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775768197.810563:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768197.810566:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294286336:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775768197.810569:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775768197.810571:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775768197.810572:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768197.810573:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775768197.810576:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b0000. 02000000:00000001:3.0:1775768197.810578:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775768197.810581:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775768197.810583:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775768197.810585:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775768197.810588:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294286336, portal 10 00000100:00000001:3.0:1775768197.810589:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775768197.810590:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611848064 : -112966097703552 : ffff994207dfed80) 00000100:00000040:3.0:1775768197.810595:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfed80 x1862022294286336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768222 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768197.810600:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775768197.810602:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775768197.810604:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775768197.810606:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294286336, offset 0 00000400:00000200:3.0:1775768197.810609:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775768197.810613:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775768197.810616:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775768197.810618:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775768197.810620:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775768197.810622:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775768197.810624:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775768197.810626:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225410:225410:256:4294967295] 192.168.202.156@tcp LPNI seq info [225406:225406:8:4294967295] 00000400:00000200:3.0:1775768197.810629:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775768197.810635:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775768197.810638:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775768197.810641:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775768197.810644:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775768197.810647:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775768197.810650:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775768197.810652:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000100:3.0:1775768197.810655:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775768197.810657:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775768197.810659:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775768197.810663:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775768197.810665:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775768197.810667:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775768197.810669:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775768197.810672:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfed80 x1862022294286336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768222 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768197.810679:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775768197.810681:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfed80 x1862022294286336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768222 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768197.810687:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768197.810689:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775768197.810692:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768197.810694:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:3.0:1775768197.810697:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775768197.810698:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300122635 00000100:00000001:3.0:1775768197.810700:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300123965 00000100:00000001:3.0:1775768197.810701:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300122635 00000100:00000001:3.0:1775768197.810702:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300123965 00000100:00000001:3.0:1775768197.810704:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775768197.810705:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768197.810706:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775768197.810708:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768197/real 1775768197] req@ffff994207dfed80 x1862022294286336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775768197.810713:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775768197.810714:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775768197.810715:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775768197.810716:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775768197.810717:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775768197.810718:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775768197.810720:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfed80 x1862022294286336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775768197.810724:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfed80 x1862022294286336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768197.810727:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775768197.810728:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768197.810729:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775768197.810730:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775768197.810732:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfed80 x1862022294286336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768197.810735:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775768197.810737:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfed80 x1862022294286336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768222 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768197.810741:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775768197.810743:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768197.810744:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775768197.810745:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768197.810745:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775768197.810747:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775768197.810748:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768197.810750:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775768197.810751:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775768197.810753:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfed80 x1862022294286336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768222 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768197.810758:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294286336:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775768197.810760:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768197.810761:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775768197.810763:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfce00 x1862022294286208/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768222 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768197.810766:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775768197.810767:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775768197.810768:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b0c00. 02000000:00000001:3.0:1775768197.810769:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775768197.810770:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775768197.810771:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:3.0:1775768197.810772:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775768197.810773:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1c800. 02000000:00000001:3.0:1775768197.810774:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775768197.810775:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775768197.810776:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfce00. 00000100:00000001:3.0:1775768197.810778:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775768197.810779:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768197.810780:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775768197.810781:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfed80 x1862022294286336/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768222 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768197.810785:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775768197.810786:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775768197.810786:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b0000. 02000000:00000001:3.0:1775768197.810787:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775768197.810788:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775768197.810788:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:3.0:1775768197.810789:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775768197.810790:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1d400. 02000000:00000001:3.0:1775768197.810792:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775768197.810793:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775768197.810793:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfed80. 00000100:00000001:3.0:1775768197.810795:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775768197.810795:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768197.810796:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768197.810798:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768197.810799:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768197.810800:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775768197.810801:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768197.810802:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775768197.810805:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfe300 x1862022294286464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768197.810811:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768197.810812:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768197.810814:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775768197.810815:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775768197.810817:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768197.810819:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294286464:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775768197.810822:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775768197.810824:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775768197.810825:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768197.810827:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775768197.810830:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b0000. 02000000:00000001:3.0:1775768197.810832:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775768197.810835:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775768197.810837:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775768197.810840:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775768197.810842:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294286464, portal 4 00000100:00000001:3.0:1775768197.810844:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775768197.810844:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611845376 : -112966097706240 : ffff994207dfe300) 00000100:00000040:3.0:1775768197.810848:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfe300 x1862022294286464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768222 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768197.810852:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775768197.810854:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775768197.810855:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775768197.810856:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294286464, offset 0 00000400:00000200:3.0:1775768197.810858:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775768197.810861:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775768197.810863:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775768197.810864:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775768197.810866:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775768197.810867:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775768197.810869:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775768197.810871:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225411:225411:256:4294967295] 192.168.202.156@tcp LPNI seq info [225407:225407:8:4294967295] 00000400:00000200:3.0:1775768197.810873:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775768197.810877:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775768197.810880:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775768197.810882:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775768197.810884:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775768197.810886:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775768197.810888:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775768197.810890:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000100:3.0:1775768197.810892:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775768197.810893:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775768197.810895:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775768197.810897:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775768197.810899:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775768197.810900:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775768197.810901:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775768197.810903:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfe300 x1862022294286464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768222 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768197.810907:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775768197.810909:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfe300 x1862022294286464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768222 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768197.810913:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768197.810914:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775768197.810915:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768197.810917:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:3.0:1775768197.810919:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775768197.810920:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300122658 00000100:00000001:3.0:1775768197.810921:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300123965 00000100:00000001:3.0:1775768197.810923:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300122658 00000100:00000001:3.0:1775768197.810924:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300123965 00000100:00000001:3.0:1775768197.810925:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775768197.810926:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768197.810927:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775768197.810929:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768197/real 1775768197] req@ffff994207dfe300 x1862022294286464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775768197.810933:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775768197.810934:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775768197.810935:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775768197.810936:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775768197.810937:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775768197.810938:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775768197.810939:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfe300 x1862022294286464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775768197.810948:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfe300 x1862022294286464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768197.810952:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775768197.810952:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768197.810953:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775768197.810954:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775768197.810956:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfe300 x1862022294286464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768197.810959:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775768197.810961:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfe300 x1862022294286464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768222 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768197.810964:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775768197.810965:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768197.810966:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775768197.810967:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768197.810968:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775768197.810970:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775768197.810971:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768197.810973:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775768197.810974:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775768197.810976:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfe300 x1862022294286464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768222 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768197.810993:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294286464:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775768197.810995:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775768197.810997:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfc700 x1862022294286592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768197.811005:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768197.811006:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768197.811007:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775768197.811009:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775768197.811010:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768197.811011:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294286592:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775768197.811013:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775768197.811014:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775768197.811015:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768197.811016:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775768197.811018:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b0c00. 02000000:00000001:3.0:1775768197.811019:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775768197.811021:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775768197.811022:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775768197.811024:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775768197.811026:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294286592, portal 4 00000100:00000001:3.0:1775768197.811027:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775768197.811028:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611838208 : -112966097713408 : ffff994207dfc700) 00000100:00000040:3.0:1775768197.811031:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfc700 x1862022294286592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768222 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768197.811034:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775768197.811036:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775768197.811037:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775768197.811038:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294286592, offset 0 00000400:00000200:3.0:1775768197.811040:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775768197.811042:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775768197.811044:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775768197.811046:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775768197.811047:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775768197.811049:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775768197.811050:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775768197.811052:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225412:225412:256:4294967295] 192.168.202.156@tcp LPNI seq info [225408:225408:8:4294967295] 00000400:00000200:3.0:1775768197.811055:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775768197.811058:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775768197.811060:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775768197.811062:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775768197.811064:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775768197.811066:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775768197.811068:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775768197.811070:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 32 secs later) 00000800:00000100:3.0:1775768197.811072:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775768197.811073:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775768197.811075:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775768197.811077:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775768197.811079:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775768197.811080:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775768197.811081:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775768197.811083:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfc700 x1862022294286592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768222 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768197.811087:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775768197.811089:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfc700 x1862022294286592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768222 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768197.811092:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768197.811093:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775768197.811094:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768197.811096:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:3.0:1775768197.811098:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775768197.811099:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300122661 00000100:00000001:3.0:1775768197.811100:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300123965 00000100:00000001:3.0:1775768197.811101:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300122661 00000100:00000001:3.0:1775768197.811102:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300123965 00000100:00000001:3.0:1775768197.811104:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775768197.811105:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768197.811106:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775768197.811108:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768197/real 1775768197] req@ffff994207dfc700 x1862022294286592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775768197.811112:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775768197.811113:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775768197.811114:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775768197.811115:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775768197.811116:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775768197.811117:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775768197.811118:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfc700 x1862022294286592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775768197.811123:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfc700 x1862022294286592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768197.811126:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775768197.811127:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768197.811128:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775768197.811129:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775768197.811131:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfc700 x1862022294286592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768197.811134:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775768197.811136:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfc700 x1862022294286592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768222 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768197.811139:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775768197.811140:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768197.811141:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775768197.811142:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768197.811143:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775768197.811145:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775768197.811146:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768197.811148:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775768197.811149:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775768197.811152:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfc700 x1862022294286592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768222 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768197.811171:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294286592:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775768197.811173:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768197.811174:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775768197.811176:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfe300 x1862022294286464/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768222 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768197.811180:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775768197.811181:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775768197.811182:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b0000. 02000000:00000001:3.0:1775768197.811184:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775768197.811185:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775768197.811185:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:3.0:1775768197.811187:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775768197.811188:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1e800. 02000000:00000001:3.0:1775768197.811189:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775768197.811190:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775768197.811191:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfe300. 00000100:00000001:3.0:1775768197.811192:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775768197.811193:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768197.811194:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775768197.811195:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfc700 x1862022294286592/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768222 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768197.811199:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775768197.811199:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775768197.811200:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b0c00. 02000000:00000001:3.0:1775768197.811201:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775768197.811201:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775768197.811202:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:3.0:1775768197.811203:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775768197.811204:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1e400. 02000000:00000001:3.0:1775768197.811205:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775768197.811206:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775768197.811207:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfc700. 00000100:00000001:3.0:1775768197.811208:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775768197.811209:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768197.811210:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768197.811211:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768197.811212:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768197.811213:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775768197.811214:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768197.873256:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768197.873262:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768197.873268:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768197.873269:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768197.873271:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768197.873272:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768197.873274:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768197.873275:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768197.873276:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768197.873277:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768197.873319:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768197.873322:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768197.873324:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768197.937263:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768197.937269:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768197.937271:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768197.937276:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768197.937280:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768197.937283:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768197.937285:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768197.937288:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768197.937291:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768198.833274:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768198.833274:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768198.833280:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768198.833280:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768198.833282:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768198.833283:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768198.833287:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:0.0:1775768198.833288:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768198.833292:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768198.833293:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768198.833296:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768198.833296:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768198.833299:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768198.833301:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768198.833303:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768198.833310:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768198.833313:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768198.833315:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768198.833410:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768198.833413:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768198.833414:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768198.833416:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775768198.833418:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768198.833419:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768198.833421:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768198.833422:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768198.833423:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768198.897240:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768198.897245:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768198.897251:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768198.897252:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768198.897255:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768198.897257:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768198.897260:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768198.897261:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768198.897264:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768198.897266:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768198.897302:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768198.897306:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768198.897309:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768198.961254:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768198.961259:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768198.961261:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768198.961265:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768198.961267:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768198.961270:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768198.961272:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768198.961274:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768198.961275:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768199.153262:0:15036:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4300123965, at 4300123968 00000100:00000040:1.0:1775768199.153262:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4300123965, at 4300123968 00000100:00000010:0.0:1775768199.153275:0:15036:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207981b40. 00000100:00000010:1.0:1775768199.153279:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207981360. 00000100:00000001:3.0:1775768199.857200:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768199.857205:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768199.857207:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768199.857212:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775768199.857216:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768199.857218:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768199.857220:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768199.857222:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768199.857224:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768199.857224:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00100000:1.0:1775768199.857228:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768199.857228:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768199.857232:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768199.857234:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768199.857240:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768199.857242:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768199.857243:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768199.857244:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768199.857248:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768199.857248:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768199.857250:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768199.857253:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775768199.857255:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768199.857257:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768199.857259:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768199.857261:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768199.857263:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768199.921260:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768199.921266:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768199.921272:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768199.921274:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768199.921276:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768199.921277:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768199.921279:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768199.921280:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768199.921282:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768199.921283:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768199.921342:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768199.921346:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768199.921349:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768199.985257:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768199.985264:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768199.985265:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768199.985270:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768199.985274:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768199.985278:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768199.985280:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768199.985282:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768199.985285:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768200.881231:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768200.881231:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768200.881238:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768200.881238:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768200.881242:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768200.881242:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768200.881246:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:1.0:1775768200.881250:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775768200.881250:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768200.881252:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768200.881256:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768200.881256:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768200.881257:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:2.0:1775768200.881261:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768200.881261:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768200.881262:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768200.881267:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768200.881267:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768200.881270:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768200.881271:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768200.881275:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768200.881276:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768200.881279:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768200.881283:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768200.881283:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768200.881286:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768200.881289:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768200.945236:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768200.945241:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768200.945246:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768200.945247:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768200.945249:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768200.945249:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768200.945251:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768200.945252:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768200.945254:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768200.945255:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768200.945299:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768200.945301:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768200.945304:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768201.009240:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768201.009245:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768201.009246:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768201.009250:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768201.009252:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768201.009255:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768201.009257:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768201.009259:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768201.009260:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768201.905215:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768201.905222:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768201.905224:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768201.905227:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768201.905228:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:2.0:1775768201.905229:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768201.905231:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768201.905233:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768201.905233:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768201.905235:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768201.905236:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768201.905237:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768201.905239:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768201.905240:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768201.905241:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768201.905244:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768201.905245:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768201.905246:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768201.905249:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768201.905250:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768201.905254:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768201.905255:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768201.905257:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768201.905260:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768201.905260:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768201.905262:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768201.905265:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768201.969250:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768201.969256:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768201.969258:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768201.969260:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768201.969264:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768201.969266:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768201.969268:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768201.969269:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768201.969271:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768201.969272:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768201.969274:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768201.969275:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768201.969277:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768201.969277:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768201.969280:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768201.969281:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768201.969320:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768201.969323:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768201.969326:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768202.033230:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768202.033236:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768202.033238:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768202.033243:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768202.033247:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768202.033250:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768202.033252:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768202.033255:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768202.033258:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768202.929231:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768202.929231:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768202.929237:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768202.929238:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768202.929239:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768202.929240:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768202.929246:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775768202.929246:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775768202.929246:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775768202.929251:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768202.929252:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1775768202.929253:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775768202.929255:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768202.929255:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768202.929256:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768202.929260:0:36:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775768202.929261:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768202.929265:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775768202.929265:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768202.929269:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768202.929270:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1775768202.929271:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5455 00000100:00000001:1.0:1775768202.929273:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768202.929277:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768202.929281:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768202.929286:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768202.929289:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768202.929292:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768202.929294:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768202.929297:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768202.929299:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768202.929301:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768202.929302:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:0.0:1775768202.929304:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768202.929306:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:0.0:1775768202.929308:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:0.0:1775768202.929309:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:0.0:1775768202.929311:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768202.929312:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768202.929314:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768202.929315:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768202.929317:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:0.0:1775768202.929319:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768202.929321:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768202.929323:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768202.929325:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768202.929327:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768202.929329:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775768202.929331:0:36:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768202.929336:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fe300. 00000020:00000040:0.0:1775768202.929340:0:36:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:0.0:1775768202.929343:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768202.929345:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768202.929348:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768202.929351:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768202.929356:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768202.929363:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220028000. 00000100:00000001:0.0:1775768202.929371:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768202.929382:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fe300 x1862022294286720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768202.929393:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768202.929398:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768202.929411:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008a1e3639] to pc [ptlrpcd_rcv+-1] req@ffff9942099fe300 x1862022294286720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768202.929420:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768202.929423:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768202.929424:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768202.929426:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768202.929428:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768202.929429:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768202.929431:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768202.929432:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768202.929433:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768202.929434:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768202.929437:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768202.929439:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775768202.929440:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768202.929441:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768202.929444:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768202.929447:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5455 00000400:00000200:0.0:1775768202.929449:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768202.929452:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768202.929453:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768202.929455:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768202.929457:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1775768202.929458:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1775768202.929459:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768202.929462:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:3.0:1775768202.929463:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1775768202.929464:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768202.929466:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000100:00000001:3.0:1775768202.929466:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000001:0.0:1775768202.929469:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768202.929471:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:0.0:1775768202.929473:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:3.0:1775768202.929474:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fe300 x1862022294286720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775768202.929475:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:0.0:1775768202.929480:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768202.929484:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768202.929488:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768202.929490:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768202.929491:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:3.0:1775768202.929492:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1775768202.929493:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00080000:0.0:1775768202.929496:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768202.929498:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768202.929498:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768202.929501:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775768202.929501:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775768202.929504:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768202.929506:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1775768202.929507:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294286720:192.168.202.156@tcp:250:kworker.0 02000000:00000001:0.0:1775768202.929510:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768202.929513:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775768202.929513:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000008:00100000:0.0:1775768202.929519:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1775768202.929519:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000008:00000001:0.0:1775768202.929521:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768202.929522:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768202.929525:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fc700. 00000020:00000040:0.0:1775768202.929529:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:3.0:1775768202.929531:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768202.929532:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775768202.929534:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1775768202.929535:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768202.929538:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768202.929539:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000010:3.0:1775768202.929540:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b0c00. 02000000:00000001:0.0:1775768202.929542:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768202.929543:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768202.929550:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002b800. 00000400:00000200:3.0:1775768202.929550:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775768202.929554:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000100:00000001:0.0:1775768202.929555:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1775768202.929560:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00100000:0.0:1775768202.929562:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fc700 x1862022294286848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:3.0:1775768202.929567:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294286720, portal 25 00000100:00000001:0.0:1775768202.929572:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775768202.929574:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768202.929576:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768202.929578:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641205504 : -112966068346112 : ffff9942099fe300) 00000100:00000040:0.0:1775768202.929582:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003283b20a] to pc [ptlrpcd_rcv+-1] req@ffff9942099fc700 x1862022294286848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775768202.929586:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fe300 x1862022294286720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768227 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768202.929608:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768202.929608:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1775768202.929612:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775768202.929612:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775768202.929615:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00080000:0.0:1775768202.929616:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775768202.929618:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768202.929619:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:3.0:1775768202.929619:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294286720, offset 0 00000100:00000001:0.0:1775768202.929623:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768202.929628:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5455 00000400:00000200:3.0:1775768202.929628:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768202.929633:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768202.929635:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768202.929638:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000400:00000200:3.0:1775768202.929638:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000040:0.0:1775768202.929642:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768202.929645:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1775768202.929646:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00000040:0.0:1775768202.929649:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768202.929652:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768202.929653:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768202.929654:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:0.0:1775768202.929656:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:3.0:1775768202.929656:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000040:0.0:1775768202.929658:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:0.0:1775768202.929660:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000400:00000200:3.0:1775768202.929663:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000020:00000001:0.0:1775768202.929664:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:0.0:1775768202.929668:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768202.929670:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1775768202.929671:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000100:00000040:0.0:1775768202.929673:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768202.929675:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768202.929678:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000400:00000200:3.0:1775768202.929678:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00080000:0.0:1775768202.929683:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768202.929685:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768202.929686:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768202.929688:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768202.929689:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768202.929690:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775768202.929691:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775768202.929693:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768202.929695:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768202.929697:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ff100. 00000020:00000040:0.0:1775768202.929699:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000400:00000200:3.0:1775768202.929699:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225413:225413:256:4294967295] 192.168.202.156@tcp LPNI seq info [225409:225409:8:4294967295] 02000000:00000001:0.0:1775768202.929702:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768202.929704:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768202.929706:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768202.929707:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:3.0:1775768202.929709:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:0.0:1775768202.929710:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768202.929715:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002bc00. 00000100:00000001:0.0:1775768202.929721:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775768202.929721:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000100:00100000:0.0:1775768202.929730:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ff100 x1862022294286976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:3.0:1775768202.929730:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775768202.929736:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000100:00000001:0.0:1775768202.929742:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000200:3.0:1775768202.929745:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:0.0:1775768202.929746:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1775768202.929749:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000100:00000040:0.0:1775768202.929754:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5e24158] to pc [ptlrpcd_rcv+-1] req@ffff9942099ff100 x1862022294286976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.1:1775768202.929756:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775768202.929761:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1775768202.929769:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775768202.929772:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000100:00000001:0.0:1775768202.929774:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775768202.929776:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00080000:0.0:1775768202.929779:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:3.0:1775768202.929782:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000100:00080000:0.0:1775768202.929785:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775768202.929788:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:3.0:1775768202.929789:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000100:00080000:0.0:1775768202.929790:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000010:3.0:1775768202.929793:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:0.0:1775768202.929795:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:3.0:1775768202.929799:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:0.0:1775768202.929800:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5455 00000100:00000200:3.0:1775768202.929806:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fe300 x1862022294286720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768227 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775768202.929808:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775768202.929820:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000400:00000200:0.0:1775768202.929824:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000040:3.0:1775768202.929825:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fe300 x1862022294286720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768227 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768202.929829:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768202.929836:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768202.929837:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768202.929838:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768202.929840:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768202.929841:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768202.929842:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:0.0:1775768202.929843:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768202.929845:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:0.0:1775768202.929846:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:0.0:1775768202.929848:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:0.0:1775768202.929849:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768202.929851:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768202.929853:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768202.929857:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768202.929861:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00000001:3.0:1775768202.929863:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768202.929864:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775768202.929866:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768202.929867:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768202.929869:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768202.929871:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768202.929872:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768202.929873:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768202.929874:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775768202.929877:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000400:00000200:3.0:1775768202.929878:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00000001:0.0:1775768202.929882:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768202.929882:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4300129085 00000100:00000001:0.0:1775768202.929888:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775768202.929888:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942079817e0. 00000100:00000010:0.0:1775768202.929893:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fd500. 00000100:00000001:3.0:1775768202.929894:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000020:00000040:0.0:1775768202.929895:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:0.0:1775768202.929898:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768202.929899:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768202.929900:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768202.929901:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768202.929902:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768202.929904:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220029400. 00000100:00000001:0.0:1775768202.929907:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768202.929910:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fd500 x1862022294287104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775768202.929910:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4300129085 00000100:00000001:0.0:1775768202.929916:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768202.929918:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775768202.929921:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207981de0. 00000100:00000040:0.0:1775768202.929922:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f246bb2b] to pc [ptlrpcd_rcv+-1] req@ffff9942099fd500 x1862022294287104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768202.929926:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1775768202.929947:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775768202.929953:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768202.929958:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1775768202.929959:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768202.929962:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775768202.929964:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768202/real 1775768202] req@ffff9942099fe300 x1862022294286720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768202.929968:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775768202.929971:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768202.929974:0:36:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768202.929979:0:36:0:(import.c:484:import_select_connection()) Process entered 00000400:00000001:3.0:1775768202.929982:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:0.0:1775768202.929983:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5455 00000400:00000001:3.0:1775768202.929985:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775768202.929989:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:0.0:1775768202.929992:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775768202.929994:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000200:0.0:1775768202.929997:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000400:00000010:3.0:1775768202.929998:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00080000:0.0:1775768202.930002:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1775768202.930003:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:0.0:1775768202.930007:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000200:3.0:1775768202.930010:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fe300 x1862022294286720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768202.930011:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768202.930016:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000200:3.0:1775768202.930025:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fe300 x1862022294286720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768202.930028:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768202.930036:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768202.930038:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:0.0:1775768202.930041:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768202.930044:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000100:00000001:3.0:1775768202.930046:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:0.0:1775768202.930047:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:0.0:1775768202.930049:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000001:3.0:1775768202.930049:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768202.930054:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:0.0:1775768202.930055:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768202.930057:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1775768202.930058:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775768202.930063:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fe300 x1862022294286720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775768202.930065:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768202.930070:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768202.930074:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:0.0:1775768202.930079:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775768202.930080:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768202.930081:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768202.930084:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768202.930086:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768202.930088:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768202.930088:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fe300 x1862022294286720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768227 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768202.930092:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775768202.930094:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:0.0:1775768202.930098:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768202.930101:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768202.930105:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000010:0.0:1775768202.930106:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ff800. 00000020:00000040:0.0:1775768202.930109:0:36:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00000100:00000001:3.0:1775768202.930110:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1775768202.930112:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768202.930114:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768202.930114:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768202.930115:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1775768202.930116:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1775768202.930117:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768202.930120:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768202.930120:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 02000000:00000010:0.0:1775768202.930123:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220028400. 00000100:00000001:3.0:1775768202.930123:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775768202.930126:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:0.0:1775768202.930128:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768202.930130:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00100000:0.0:1775768202.930134:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ff800 x1862022294287232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775768202.930134:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775768202.930141:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fe300 x1862022294286720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768227 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768202.930180:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768202.930181:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768202.930186:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000aa4e8b71] to pc [ptlrpcd_rcv+-1] req@ffff9942099ff800 x1862022294287232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768202.930190:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294286720:192.168.202.156@tcp:250:kworker.0 00000100:00000001:3.0:1775768202.930198:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768202.930199:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768202.930200:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775768202.930204:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fe300 x1862022294286720/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768227 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775768202.930205:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5465) 00000100:00000001:3.0:1775768202.930212:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775768202.930213:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775768202.930216:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b0c00. 02000000:00000001:3.0:1775768202.930223:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775768202.930224:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775768202.930227:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:3.0:1775768202.930228:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775768202.930230:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028000. 02000000:00000001:3.0:1775768202.930232:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775768202.930233:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775768202.930234:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fe300. 00000100:00000001:3.0:1775768202.930237:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775768202.930237:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768202.930239:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768202.930243:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768202.930244:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768202.930246:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775768202.930247:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768202.930248:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775768202.930250:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fc700 x1862022294286848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768202.930254:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768202.930255:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768202.930256:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775768202.930257:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775768202.930259:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768202.930260:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294286848:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775768202.930262:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775768202.930264:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775768202.930265:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768202.930266:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775768202.930268:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b0c00. 02000000:00000001:3.0:1775768202.930269:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775768202.930272:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775768202.930274:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775768202.930276:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775768202.930278:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294286848, portal 10 00000100:00000001:3.0:1775768202.930279:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775768202.930280:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641198336 : -112966068353280 : ffff9942099fc700) 00000100:00000040:3.0:1775768202.930284:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fc700 x1862022294286848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768227 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768202.930288:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775768202.930289:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775768202.930290:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775768202.930292:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294286848, offset 0 00000400:00000200:3.0:1775768202.930294:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775768202.930296:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775768202.930299:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775768202.930301:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775768202.930303:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775768202.930304:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775768202.930311:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775768202.930314:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225414:225414:256:4294967295] 192.168.202.156@tcp LPNI seq info [225410:225410:8:4294967295] 00000400:00000200:3.0:1775768202.930321:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775768202.930327:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775768202.930331:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775768202.930335:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775768202.930338:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775768202.930341:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775768202.930345:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775768202.930347:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1775768202.930350:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775768202.930352:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775768202.930354:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775768202.930357:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775768202.930360:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775768202.930362:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775768202.930364:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775768202.930367:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fc700 x1862022294286848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768227 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768202.930387:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775768202.930390:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fc700 x1862022294286848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768227 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768202.930397:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768202.930398:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775768202.930402:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768202.930405:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:3.0:1775768202.930408:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775768202.930410:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300127763 00000100:00000001:3.0:1775768202.930413:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300129085 00000100:00000001:3.0:1775768202.930415:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300127763 00000100:00000001:3.0:1775768202.930416:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300129085 00000100:00000001:3.0:1775768202.930417:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775768202.930419:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768202.930420:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775768202.930423:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768202/real 1775768202] req@ffff9942099fc700 x1862022294286848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775768202.930427:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775768202.930428:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775768202.930429:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775768202.930430:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775768202.930432:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775768202.930433:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775768202.930434:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fc700 x1862022294286848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775768202.930439:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fc700 x1862022294286848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768202.930443:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775768202.930444:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768202.930445:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775768202.930446:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775768202.930448:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fc700 x1862022294286848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768202.930451:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775768202.930453:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fc700 x1862022294286848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768227 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768202.930457:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775768202.930458:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768202.930459:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775768202.930460:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768202.930461:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775768202.930462:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775768202.930464:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768202.930465:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775768202.930467:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775768202.930469:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fc700 x1862022294286848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768227 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768202.930473:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294286848:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775768202.930475:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775768202.930476:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ff100 x1862022294286976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768202.930480:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768202.930481:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768202.930482:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775768202.930484:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775768202.930485:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768202.930487:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294286976:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775768202.930489:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775768202.930490:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775768202.930491:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768202.930492:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775768202.930494:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b0000. 02000000:00000001:3.0:1775768202.930495:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775768202.930497:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775768202.930499:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775768202.930500:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775768202.930502:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294286976, portal 10 00000100:00000001:3.0:1775768202.930503:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775768202.930504:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641209088 : -112966068342528 : ffff9942099ff100) 00000100:00000040:3.0:1775768202.930508:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ff100 x1862022294286976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768227 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768202.930511:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775768202.930513:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775768202.930514:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775768202.930515:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294286976, offset 0 00000400:00000200:3.0:1775768202.930517:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775768202.930519:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775768202.930521:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775768202.930523:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775768202.930525:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775768202.930526:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775768202.930528:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775768202.930530:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225415:225415:256:4294967295] 192.168.202.156@tcp LPNI seq info [225411:225411:8:4294967295] 00000400:00000200:3.0:1775768202.930532:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775768202.930536:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775768202.930539:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775768202.930541:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775768202.930543:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775768202.930545:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775768202.930547:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775768202.930549:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1775768202.930551:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775768202.930552:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775768202.930554:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775768202.930556:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775768202.930557:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775768202.930558:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775768202.930560:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775768202.930561:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ff100 x1862022294286976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768227 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768202.930566:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775768202.930568:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ff100 x1862022294286976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768227 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768202.930571:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768202.930572:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775768202.930574:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768202.930576:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:3.0:1775768202.930577:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775768202.930579:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300127755 00000100:00000001:3.0:1775768202.930580:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300129085 00000100:00000001:3.0:1775768202.930581:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300127755 00000100:00000001:3.0:1775768202.930582:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300129085 00000100:00000001:3.0:1775768202.930583:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775768202.930584:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768202.930585:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775768202.930587:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768202/real 1775768202] req@ffff9942099ff100 x1862022294286976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775768202.930591:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775768202.930592:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775768202.930593:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775768202.930594:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775768202.930595:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775768202.930596:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775768202.930598:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ff100 x1862022294286976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775768202.930602:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ff100 x1862022294286976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768202.930605:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775768202.930606:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768202.930607:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775768202.930607:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775768202.930609:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ff100 x1862022294286976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768202.930612:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775768202.930614:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ff100 x1862022294286976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768227 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768202.930618:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775768202.930619:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768202.930620:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775768202.930620:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768202.930622:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775768202.930623:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775768202.930625:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768202.930627:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775768202.930628:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775768202.930630:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ff100 x1862022294286976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768227 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768202.930638:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294286976:192.168.202.156@tcp:38:kworker.0 00000100:00000001:3.0:1775768202.930640:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775768202.930642:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fd500 x1862022294287104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768202.930646:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768202.930647:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768202.930648:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775768202.930650:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775768202.930651:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768202.930654:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294287104:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775768202.930657:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775768202.930658:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775768202.930659:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768202.930661:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775768202.930664:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b2800. 02000000:00000001:3.0:1775768202.930665:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775768202.930669:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775768202.930671:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775768202.930674:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775768202.930676:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294287104, portal 4 00000100:00000001:3.0:1775768202.930678:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775768202.930679:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641201920 : -112966068349696 : ffff9942099fd500) 00000100:00000040:3.0:1775768202.930684:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fd500 x1862022294287104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768227 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768202.930691:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775768202.930693:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775768202.930695:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775768202.930697:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294287104, offset 0 00000400:00000200:3.0:1775768202.930700:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775768202.930704:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775768202.930706:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775768202.930708:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775768202.930710:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775768202.930711:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775768202.930713:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775768202.930714:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225416:225416:256:4294967295] 192.168.202.156@tcp LPNI seq info [225412:225412:8:4294967295] 00000400:00000200:3.0:1775768202.930717:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775768202.930721:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775768202.930723:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775768202.930725:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775768202.930727:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775768202.930729:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775768202.930731:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775768202.930733:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1775768202.930735:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775768202.930736:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775768202.930738:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775768202.930740:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775768202.930742:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775768202.930743:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775768202.930744:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775768202.930746:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fd500 x1862022294287104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768227 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768202.930751:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775768202.930752:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fd500 x1862022294287104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768227 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768202.930756:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768202.930757:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775768202.930759:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768202.930761:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:3.0:1775768202.930763:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775768202.930764:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300127778 00000100:00000001:3.0:1775768202.930765:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300129085 00000100:00000001:3.0:1775768202.930767:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300127778 00000100:00000001:3.0:1775768202.930768:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300129085 00000100:00000001:3.0:1775768202.930769:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775768202.930770:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768202.930771:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775768202.930773:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768202/real 1775768202] req@ffff9942099fd500 x1862022294287104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775768202.930777:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775768202.930778:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775768202.930779:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775768202.930780:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775768202.930781:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775768202.930782:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775768202.930784:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fd500 x1862022294287104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775768202.930791:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fd500 x1862022294287104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768202.930797:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775768202.930798:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768202.930800:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775768202.930801:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775768202.930804:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fd500 x1862022294287104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768202.930810:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775768202.930814:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fd500 x1862022294287104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768227 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768202.930820:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775768202.930823:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768202.930824:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775768202.930825:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768202.930827:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775768202.930830:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775768202.930833:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768202.930836:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775768202.930838:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775768202.930840:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fd500 x1862022294287104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768227 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768202.930845:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294287104:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775768202.930847:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775768202.930849:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ff800 x1862022294287232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768202.930853:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768202.930854:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768202.930855:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775768202.930856:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775768202.930857:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768202.930859:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294287232:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775768202.930860:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775768202.930862:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775768202.930863:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768202.930864:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775768202.930866:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9942054b2c00. 02000000:00000001:3.0:1775768202.930867:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775768202.930869:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:3.0:1775768202.930870:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99420780b5d8. 00000400:00000010:3.0:1775768202.930872:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5ebb0. 00000100:00000200:3.0:1775768202.930874:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294287232, portal 4 00000100:00000001:3.0:1775768202.930876:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775768202.930877:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641210880 : -112966068340736 : ffff9942099ff800) 00000100:00000040:3.0:1775768202.930880:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ff800 x1862022294287232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768227 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768202.930884:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775768202.930885:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:3.0:1775768202.930886:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994207f5e6e8. 00000100:00000200:3.0:1775768202.930888:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294287232, offset 0 00000400:00000200:3.0:1775768202.930890:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775768202.930892:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775768202.930894:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:3.0:1775768202.930896:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775768202.930897:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:3.0:1775768202.930899:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775768202.930900:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:3.0:1775768202.930902:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225417:225417:256:4294967295] 192.168.202.156@tcp LPNI seq info [225413:225413:8:4294967295] 00000400:00000200:3.0:1775768202.930904:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775768202.930908:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:3.0:1775768202.930910:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775768202.930912:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942201b8f00. 00000800:00000200:3.0:1775768202.930914:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.0:1775768202.930916:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775768202.930918:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:3.1:1775768202.930919:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1775768202.930921:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:3.0:1775768202.930923:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942201b8f00. 00000400:00000200:3.0:1775768202.930924:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775768202.930926:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775768202.930928:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5e6e8 00000400:00000010:3.0:1775768202.930929:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5e6e8. 00000100:00000001:3.0:1775768202.930942:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775768202.930946:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ff800 x1862022294287232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768227 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768202.930954:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775768202.930956:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ff800 x1862022294287232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768227 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768202.930960:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768202.930961:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775768202.930963:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768202.930965:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:3.0:1775768202.930967:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775768202.930968:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300127781 00000100:00000001:3.0:1775768202.930970:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300129085 00000100:00000001:3.0:1775768202.930971:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300127781 00000100:00000001:3.0:1775768202.930972:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300129085 00000100:00000001:3.0:1775768202.930974:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775768202.930975:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768202.930976:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775768202.930977:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768202/real 1775768202] req@ffff9942099ff800 x1862022294287232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775768202.930984:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775768202.930985:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775768202.930986:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99420780b5d8. 00000400:00000200:3.0:1775768202.930988:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994207f5ebb0 00000400:00000010:3.0:1775768202.930989:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994207f5ebb0. 00000100:00000001:3.0:1775768202.930991:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775768202.930994:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ff800 x1862022294287232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775768202.931001:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ff800 x1862022294287232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768202.931006:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775768202.931008:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768202.931010:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775768202.931011:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775768202.931014:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ff800 x1862022294287232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768202.931020:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775768202.931023:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ff800 x1862022294287232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768227 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768202.931030:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775768202.931033:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768202.931034:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775768202.931035:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768202.931037:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775768202.931040:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775768202.931042:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768202.931045:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:3.0:1775768202.931047:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775768202.931049:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ff800 x1862022294287232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768227 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768202.931054:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294287232:192.168.202.156@tcp:8:kworker.0 00000100:00000001:3.0:1775768202.931057:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768202.931058:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775768202.931060:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fc700 x1862022294286848/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768227 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768202.931066:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775768202.931067:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775768202.931069:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b0c00. 02000000:00000001:3.0:1775768202.931072:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775768202.931073:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775768202.931074:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:3.0:1775768202.931076:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775768202.931077:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002b800. 02000000:00000001:3.0:1775768202.931080:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775768202.931081:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775768202.931082:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fc700. 00000100:00000001:3.0:1775768202.931085:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775768202.931086:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768202.931087:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775768202.931090:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ff100 x1862022294286976/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768227 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768202.931096:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775768202.931098:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775768202.931099:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b0000. 02000000:00000001:3.0:1775768202.931101:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775768202.931102:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775768202.931103:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:3.0:1775768202.931105:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775768202.931105:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002bc00. 02000000:00000001:3.0:1775768202.931107:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775768202.931107:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775768202.931108:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ff100. 00000100:00000001:3.0:1775768202.931110:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775768202.931111:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768202.931112:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775768202.931114:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fd500 x1862022294287104/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768227 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768202.931118:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775768202.931118:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775768202.931119:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b2800. 02000000:00000001:3.0:1775768202.931120:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775768202.931121:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775768202.931121:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:3.0:1775768202.931122:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775768202.931123:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029400. 02000000:00000001:3.0:1775768202.931124:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775768202.931125:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775768202.931126:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fd500. 00000100:00000001:3.0:1775768202.931127:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775768202.931128:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768202.931129:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775768202.931130:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ff800 x1862022294287232/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768227 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768202.931133:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775768202.931134:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775768202.931135:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9942054b2c00. 02000000:00000001:3.0:1775768202.931136:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775768202.931136:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775768202.931137:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:3.0:1775768202.931138:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775768202.931138:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028400. 02000000:00000001:3.0:1775768202.931139:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775768202.931140:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775768202.931140:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ff800. 00000100:00000001:3.0:1775768202.931141:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775768202.931142:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768202.931143:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768202.931145:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768202.931146:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768202.931147:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775768202.931148:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768202.993251:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768202.993255:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768202.993260:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768202.993261:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768202.993264:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768202.993265:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768202.993267:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768202.993268:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768202.993270:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768202.993270:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768202.993298:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768202.993300:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768202.993303:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768203.057259:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768203.057265:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768203.057267:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768203.057272:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768203.057275:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768203.057279:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768203.057282:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768203.057284:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768203.057287:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768203.953233:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768203.953240:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768203.953240:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768203.953243:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768203.953244:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768203.953246:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768203.953251:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:2.0:1775768203.953251:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768203.953255:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768203.953256:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768203.953256:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768203.953259:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768203.953260:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768203.953261:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768203.953261:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768203.953265:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768203.953265:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768203.953269:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768203.953270:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768203.953271:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768203.953274:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768203.953275:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768203.953279:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768203.953280:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768203.953283:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768203.953286:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768203.953289:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768204.017268:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768204.017275:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768204.017281:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768204.017283:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768204.017287:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768204.017288:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768204.017291:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768204.017292:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768204.017295:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768204.017296:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768204.017326:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768204.017334:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768204.017339:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768204.081210:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768204.081215:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768204.081216:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768204.081220:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768204.081223:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768204.081226:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768204.081227:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768204.081229:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768204.081230:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768204.273259:0:15036:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4300129085, at 4300129088 00000100:00000040:1.0:1775768204.273259:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4300129085, at 4300129088 00000100:00000010:0.0:1775768204.273271:0:15036:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942079817e0. 00000100:00000010:1.0:1775768204.273272:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207981de0. 00000100:00000001:0.0:1775768204.977257:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768204.977257:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768204.977257:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768204.977264:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768204.977265:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768204.977265:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768204.977269:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768204.977270:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768204.977270:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768204.977279:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:1.0:1775768204.977279:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:2.0:1775768204.977279:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768204.977286:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768204.977287:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768204.977287:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768204.977292:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768204.977292:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768204.977293:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768204.977300:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768204.977303:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768204.977306:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768204.977308:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768204.977311:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768204.977314:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768204.977315:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768204.977318:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768204.977320:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768205.041238:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768205.041245:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768205.041252:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768205.041253:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768205.041257:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768205.041258:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768205.041262:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768205.041263:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768205.041267:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768205.041268:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768205.041319:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768205.041322:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768205.041326:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768205.105224:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768205.105229:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768205.105231:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768205.105236:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768205.105240:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768205.105245:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768205.105247:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768205.105250:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768205.105253:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768206.001226:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768206.001231:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768206.001234:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768206.001239:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775768206.001243:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768206.001246:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768206.001249:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768206.001251:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768206.001254:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768206.001292:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768206.001297:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768206.001299:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768206.001302:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768206.001306:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768206.001308:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768206.001310:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768206.001312:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768206.001314:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768206.001328:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768206.001333:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768206.001335:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768206.001339:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775768206.001343:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768206.001354:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768206.001357:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768206.001359:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768206.001362:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768206.065249:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768206.065254:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768206.065258:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768206.065259:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768206.065262:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768206.065263:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768206.065266:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768206.065267:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768206.065269:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768206.065270:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768206.065345:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768206.065349:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768206.065352:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768206.129259:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768206.129265:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768206.129268:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768206.129274:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768206.129278:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768206.129282:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768206.129285:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768206.129287:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768206.129290:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768207.025271:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768207.025271:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768207.025279:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768207.025280:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768207.025283:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768207.025284:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768207.025291:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775768207.025292:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768207.025299:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768207.025299:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768207.025305:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768207.025305:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768207.025308:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768207.025312:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768207.025313:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768207.025316:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768207.025318:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768207.025320:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768207.025323:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768207.025324:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768207.025325:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768207.025327:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768207.025329:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768207.025334:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768207.025341:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768207.025344:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768207.025348:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768207.089238:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768207.089243:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768207.089249:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768207.089252:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768207.089254:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768207.089258:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768207.089261:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768207.089264:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768207.089266:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768207.089268:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768207.089272:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768207.089274:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768207.089275:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768207.089277:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768207.089278:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768207.089280:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768207.089281:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768207.089284:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768207.089287:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768207.089289:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768207.089290:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768207.089292:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768207.089294:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768207.089295:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768207.089296:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768207.089298:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768207.089301:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768207.089303:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768207.089304:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768207.089306:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768207.089307:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768207.089309:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768207.089310:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768207.089312:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768207.089359:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768207.089362:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768207.089364:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768207.153249:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768207.153255:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768207.153257:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768207.153261:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768207.153264:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768207.153267:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768207.153269:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768207.153271:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768207.153273:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768208.049222:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768208.049229:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768208.049231:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768208.049236:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:1.0:1775768208.049236:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775768208.049241:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768208.049242:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768208.049245:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768208.049247:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768208.049248:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768208.049250:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000080:00000001:3.0:1775768208.049251:0:13471:0:(rw.c:1481:ll_writepages()) Process entered 00000100:00000001:0.0:1775768208.049252:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768208.049253:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768208.049256:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768208.049257:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:3.0:1775768208.049258:0:13471:0:(file.c:5250:cl_sync_file_range()) Process entered 00000100:00000001:0.0:1775768208.049260:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768208.049263:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1775768208.049264:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00000001:0.0:1775768208.049266:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1775768208.049274:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff99420693e000. 00000100:00080000:2.0:1775768208.049275:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768208.049281:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000020:00000010:3.0:1775768208.049281:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcf000. 00000100:00000001:2.0:1775768208.049285:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775768208.049286:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000020:00000010:3.0:1775768208.049287:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210dcf200. 00000100:00080000:2.0:1775768208.049288:0:36:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768208.049290:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768208.049293:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768208.049293:0:36:0:(import.c:484:import_select_connection()) Process entered 00000020:00000010:3.0:1775768208.049293:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207981e40. 00000100:00100000:1.0:1775768208.049296:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768208.049299:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768208.049301:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1775768208.049302:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5460 00000008:00000010:3.0:1775768208.049302:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994208b50000. 00000100:00000001:1.0:1775768208.049304:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768208.049307:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768208.049309:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768208.049315:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000010:3.0:1775768208.049318:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994218000300. 00000400:00000200:2.0:1775768208.049322:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00008000:00000010:3.0:1775768208.049324:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994207e08e20. 00000100:00080000:2.0:1775768208.049325:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775768208.049329:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000080:00000010:3.0:1775768208.049330:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99420541b1f8. 00000100:00000001:2.0:1775768208.049331:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768208.049334:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000080:00000010:3.0:1775768208.049334:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077c0248. 00000100:00000001:2.0:1775768208.049337:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000010:3.0:1775768208.049338:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcf800. 00000020:00000001:2.0:1775768208.049340:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775768208.049342:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000008:00000010:3.0:1775768208.049342:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942086f8420. 00000020:00000001:2.0:1775768208.049345:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00020000:00000010:3.0:1775768208.049345:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994207e8e400. 00000020:00000040:2.0:1775768208.049348:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00008000:00000010:3.0:1775768208.049349:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994209270a90. 00000020:00000001:2.0:1775768208.049350:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000080:00000010:3.0:1775768208.049352:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942090e33c0. 00000020:00000001:2.0:1775768208.049353:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:3.0:1775768208.049355:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107590086664 : -112966119464952 : ffff99420693e008) 00000100:00000040:2.0:1775768208.049356:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768208.049358:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00001000:3.0:1775768208.049358:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff99420693e000 00000100:00000040:2.0:1775768208.049360:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768208.049362:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775768208.049362:0:13471:0:(cl_object.c:151:cl_object_top()) ffff99420791a5f8 -> ffff99420791a5f8 00000020:00000040:2.0:1775768208.049365:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000020:00000001:3.0:1775768208.049365:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:3.0:1775768208.049367:0:13471:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000100:00080000:2.0:1775768208.049368:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000080:00200000:3.0:1775768208.049370:0:13471:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x10176:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000100:00000001:2.0:1775768208.049371:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768208.049373:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 02000000:00000001:2.0:1775768208.049375:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000020:00000001:3.0:1775768208.049375:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:3.0:1775768208.049377:0:13471:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768208.049378:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768208.049380:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:3.0:1775768208.049381:0:13471:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x10176:0x0]io ffff9942077c02a8 type 6 ignore/verify layout 1/0 00000100:00000001:2.0:1775768208.049383:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00020000:00000002:3.0:1775768208.049384:0:13471:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207dfb6c8) owner 0000000000000000/ffff994220080000 00000100:00000001:2.0:1775768208.049385:0:36:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768208.049388:0:13471:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:3.0:1775768208.049390:0:13471:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00000100:00000010:2.0:1775768208.049391:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dff100. 00020000:00000001:3.0:1775768208.049393:0:13471:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00000020:00000040:2.0:1775768208.049394:0:36:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 00020000:00000001:3.0:1775768208.049394:0:13471:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768208.049397:0:13471:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768208.049398:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00020000:00000001:3.0:1775768208.049399:0:13471:0:(lov_io.c:203:lov_io_subio_init()) Process entered 02000000:00000001:2.0:1775768208.049400:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768208.049401:0:13471:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768208.049402:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000100:00000001:2.0:1775768208.049403:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000020:00000001:3.0:1775768208.049404:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 02000000:00000001:2.0:1775768208.049405:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00020000:00000001:3.0:1775768208.049405:0:13471:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00020000:00000002:3.0:1775768208.049406:0:13471:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207dfb6c8) owner 0000000000000000/ffff994220080000 00000020:00000001:3.0:1775768208.049409:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768208.049410:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768208.049412:0:13471:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:3.0:1775768208.049413:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00020000:00000001:3.0:1775768208.049415:0:13471:0:(lov_io.c:836:lov_io_iter_init()) Process entered 02000000:00000010:2.0:1775768208.049417:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1f400. 00020000:00200000:3.0:1775768208.049418:0:13471:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00000002:3.0:1775768208.049421:0:13471:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00000100:00000001:2.0:1775768208.049423:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768208.049424:0:13471:0:(lov_io.c:155:lov_sub_get()) Process entered 00020000:00000001:3.0:1775768208.049426:0:13471:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000020:00000001:3.0:1775768208.049427:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:3.0:1775768208.049430:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff99420693ecf0. 00000100:00100000:2.0:1775768208.049432:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dff100 x1862022294287360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000010:3.0:1775768208.049433:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcf600. 00000020:00000010:3.0:1775768208.049436:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210dce400. 00000020:00000010:3.0:1775768208.049438:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207981900. 00000008:00000010:3.0:1775768208.049442:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994208b51650. 00000100:00000001:2.0:1775768208.049443:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00020000:00000010:3.0:1775768208.049445:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994218000000. 00000100:00000001:2.0:1775768208.049447:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:3.0:1775768208.049447:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994207e0ade8. 00000080:00000010:3.0:1775768208.049451:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99420541b0a8. 00000100:00000040:2.0:1775768208.049453:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3e64ae1] to pc [ptlrpcd_rcv+-1] req@ffff994207dff100 x1862022294287360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000010:3.0:1775768208.049453:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077c1da8. 00000020:00000010:3.0:1775768208.049456:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dce800. 00000008:00000010:3.0:1775768208.049459:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942086f8160. 00020000:00000010:3.0:1775768208.049462:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994207e8ed00. 00008000:00000010:3.0:1775768208.049464:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994209270ef0. 00000080:00000010:3.0:1775768208.049468:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942090e3960. 00000020:00000001:3.0:1775768208.049471:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107590089976 : -112966119461640 : ffff99420693ecf8) 00000020:00001000:3.0:1775768208.049473:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff99420693ecf0 00000020:00000001:3.0:1775768208.049475:0:13471:0:(cl_object.c:151:cl_object_top()) ffff99420791ae90 -> ffff99420791a5f8 00000100:00000001:2.0:1775768208.049476:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768208.049477:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000100:00080000:2.0:1775768208.049480:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000001:3.0:1775768208.049480:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:3.0:1775768208.049482:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00080000:2.0:1775768208.049484:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000020:00000001:3.0:1775768208.049484:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768208.049486:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00020000:00000001:3.0:1775768208.049486:0:13471:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768208.049488:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00020000:00000001:3.0:1775768208.049488:0:13471:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107612435560 : -112966097116056 : ffff994207e8e468) 00000100:00000001:2.0:1775768208.049492:0:36:0:(import.c:484:import_select_connection()) Process entered 00000020:00000001:3.0:1775768208.049492:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000020:00000001:3.0:1775768208.049493:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768208.049495:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5460 00020000:00000001:3.0:1775768208.049496:0:13471:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768208.049498:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768208.049499:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768208.049500:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1775768208.049501:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00020000:00000001:3.0:1775768208.049503:0:13471:0:(lov_io.c:1108:lov_io_lock()) Process entered 00000100:00000001:0.0:1775768208.049504:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000400:00000200:2.0:1775768208.049505:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00020000:00000001:3.0:1775768208.049505:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000001:0.0:1775768208.049506:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1775768208.049508:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00000100:00080000:2.0:1775768208.049511:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000020:00000001:3.0:1775768208.049511:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:3.0:1775768208.049513:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768208.049515:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dff100 x1862022294287360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775768208.049516:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768208.049517:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00020000:00000001:3.0:1775768208.049521:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768208.049522:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00020000:00000001:3.0:1775768208.049524:0:13471:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768208.049525:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000001:3.0:1775768208.049526:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000100:00000001:2.0:1775768208.049528:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775768208.049528:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768208.049531:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000001:3.0:1775768208.049531:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775768208.049534:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:3.0:1775768208.049535:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00000020:00000001:2.0:1775768208.049537:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00020000:00000001:3.0:1775768208.049539:0:13471:0:(lov_io.c:1114:lov_io_start()) Process entered 00000020:00000040:2.0:1775768208.049541:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00020000:00000001:3.0:1775768208.049541:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775768208.049543:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00000100:00000001:0.0:1775768208.049545:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000001:2.0:1775768208.049546:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000008:00000001:3.0:1775768208.049546:0:13471:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000100:00000001:0.0:1775768208.049548:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768208.049549:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000008:00000001:3.0:1775768208.049549:0:13471:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 02000000:00000001:0.0:1775768208.049552:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:2.0:1775768208.049553:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 02000000:00000001:0.0:1775768208.049555:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768208.049558:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1775768208.049559:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768208.049564:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000008:00000001:3.0:1775768208.049564:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000100:00100000:0.0:1775768208.049565:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294287360:192.168.202.156@tcp:250:kworker.0 00000008:00000020:3.0:1775768208.049568:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000100:00000001:2.0:1775768208.049570:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000008:00000001:3.0:1775768208.049570:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768208.049572:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000040:2.0:1775768208.049573:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000008:00000001:3.0:1775768208.049574:0:13471:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 02000000:00000001:0.0:1775768208.049576:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000008:00000001:3.0:1775768208.049576:0:13471:0:(osc_cache.c:2287:osc_next_obj()) Process entered 02000000:00000001:0.0:1775768208.049579:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768208.049579:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000008:00000001:3.0:1775768208.049580:0:13471:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107754940544 : -112965954611072 : ffff994210675880) 00000100:00000001:2.0:1775768208.049583:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768208.049586:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1775768208.049586:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000008:00000020:3.0:1775768208.049586:0:13471:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994210675880 ready 0|-|- wr 1|-|+ rd 0|- 10 in flight 00000008:00000001:3.0:1775768208.049590:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 02000000:00000010:0.0:1775768208.049591:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220029800. 02000000:00000001:2.0:1775768208.049593:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00000020:3.0:1775768208.049593:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 02000000:00000001:0.0:1775768208.049594:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1775768208.049596:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1775768208.049597:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768208.049600:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00000001:3.0:1775768208.049600:0:13471:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000400:00000200:0.0:1775768208.049603:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00100000:2.0:1775768208.049604:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000020:3.0:1775768208.049604:0:13471:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994218232fc0 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000400:00000010:0.0:1775768208.049608:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000008:00000001:2.0:1775768208.049609:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1775768208.049609:0:13471:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768208.049613:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1775768208.049614:0:13471:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775768208.049615:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000020:00000001:3.0:1775768208.049616:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775768208.049618:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfdc00. 00020000:00000001:3.0:1775768208.049620:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775768208.049621:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000100:00000200:0.0:1775768208.049622:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294287360, portal 25 00020000:00000001:3.0:1775768208.049622:0:13471:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768208.049624:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000020:00000001:3.0:1775768208.049624:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768208.049627:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768208.049628:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768208.049629:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000020:00000001:3.0:1775768208.049629:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00000100:00000001:0.0:1775768208.049630:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611848960 : -112966097702656 : ffff994207dff100) 02000000:00000001:2.0:1775768208.049632:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00020000:00000001:3.0:1775768208.049632:0:13471:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 02000000:00000001:2.0:1775768208.049635:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768208.049635:0:13471:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000020:00000001:3.0:1775768208.049637:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00000100:00000040:0.0:1775768208.049639:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dff100 x1862022294287360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768233 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1775768208.049640:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1f000. 00000020:00000001:3.0:1775768208.049640:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00020000:00000001:3.0:1775768208.049643:0:13471:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768208.049646:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768208.049647:0:13471:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000020:00000001:3.0:1775768208.049649:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00000020:00000001:3.0:1775768208.049652:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000100:00100000:2.0:1775768208.049656:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfdc00 x1862022294287488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00020000:00000001:3.0:1775768208.049656:0:13471:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00000100:00000001:0.0:1775768208.049657:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00020000:00000001:3.0:1775768208.049658:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775768208.049660:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000020:00000001:3.0:1775768208.049663:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000100:00000040:0.0:1775768208.049664:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00020000:00000001:3.0:1775768208.049665:0:13471:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775768208.049669:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00020000:00000001:3.0:1775768208.049669:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768208.049672:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00020000:00000001:3.0:1775768208.049672:0:13471:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000100:00000200:0.0:1775768208.049673:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294287360, offset 0 00000020:00000001:3.0:1775768208.049674:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000100:00000001:2.0:1775768208.049676:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768208.049678:0:13471:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000020:00000001:3.0:1775768208.049679:0:13471:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000001:3.0:1775768208.049681:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000100:00000040:2.0:1775768208.049683:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003e25603c] to pc [ptlrpcd_rcv+-1] req@ffff994207dfdc00 x1862022294287488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:3.0:1775768208.049683:0:13471:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00000400:00000200:0.0:1775768208.049684:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00020000:00000001:3.0:1775768208.049685:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775768208.049688:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000001:3.0:1775768208.049689:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00020000:00000001:3.0:1775768208.049692:0:13471:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768208.049696:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:3.0:1775768208.049696:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768208.049698:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768208.049699:0:13471:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000020:00000001:3.0:1775768208.049700:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000100:00080000:2.0:1775768208.049702:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000001:3.0:1775768208.049702:0:13471:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768208.049704:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000020:00000001:3.0:1775768208.049706:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00000100:00080000:2.0:1775768208.049708:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00020000:00000001:3.0:1775768208.049709:0:13471:0:(lov_io.c:675:lov_io_fini()) Process entered 00000100:00000001:2.0:1775768208.049712:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00020000:00000001:3.0:1775768208.049712:0:13471:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000400:00000200:0.0:1775768208.049713:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:2.0:1775768208.049714:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000020:00000001:3.0:1775768208.049714:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:3.0:1775768208.049717:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000400:00000200:0.0:1775768208.049719:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000100:00000001:2.0:1775768208.049720:0:36:0:(import.c:484:import_select_connection()) Process entered 00000020:00001000:3.0:1775768208.049720:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff99420693ecf0 00000400:00000200:0.0:1775768208.049724:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000020:00000010:3.0:1775768208.049725:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210dce400. 00000100:00080000:2.0:1775768208.049726:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5460 00000400:00000200:0.0:1775768208.049730:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000020:00000010:3.0:1775768208.049732:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207981900. 00000400:00000200:2.0:1775768208.049733:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768208.049738:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225418:225418:256:4294967295] 192.168.202.156@tcp LPNI seq info [225414:225414:8:4294967295] 00000008:00000010:3.0:1775768208.049739:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994208b51650. 00020000:00000010:3.0:1775768208.049746:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994218000000. 00000400:00000200:0.0:1775768208.049747:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00008000:00000010:3.0:1775768208.049752:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994207e0ade8. 00000080:00000010:3.0:1775768208.049756:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99420541b0a8. 00000080:00000010:3.0:1775768208.049762:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077c1da8. 00000400:00000200:0.0:1775768208.049765:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000020:00000010:3.0:1775768208.049767:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcf600. 00000400:00000200:2.0:1775768208.049772:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000008:00000010:3.0:1775768208.049774:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942086f8160. 00000100:00080000:2.0:1775768208.049776:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00020000:00000010:3.0:1775768208.049778:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994207e8ed00. 00000800:00000200:0.0:1775768208.049780:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00000040:2.0:1775768208.049782:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000800:00000010:0.0:1775768208.049787:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000100:00000001:2.0:1775768208.049787:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00008000:00000010:3.0:1775768208.049790:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994209270ef0. 00000100:00000040:2.0:1775768208.049791:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768208.049793:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000080:00000010:3.0:1775768208.049795:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942090e3960. 00000020:00000001:2.0:1775768208.049797:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775768208.049800:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000010:3.0:1775768208.049800:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dce800. 00000800:00000200:0.0:1775768208.049801:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000001:2.0:1775768208.049803:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000010:3.0:1775768208.049803:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff99420693ecf0. 00000020:00000040:2.0:1775768208.049807:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00020000:00000001:3.0:1775768208.049807:0:13471:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00000800:00000200:0.0:1775768208.049808:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000020:00000001:2.0:1775768208.049809:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00020000:00000001:3.0:1775768208.049810:0:13471:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:3.0:1775768208.049813:0:13471:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000020:00000001:2.0:1775768208.049814:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000080:00200000:3.0:1775768208.049815:0:13471:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x10176:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000800:00000200:0.1:1775768208.049819:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000080:00000001:3.0:1775768208.049821:0:13471:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000100:00000040:2.0:1775768208.049822:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768208.049824:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:3.0:1775768208.049824:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000800:00000200:0.1:1775768208.049824:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000100:00000040:2.0:1775768208.049829:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00001000:3.0:1775768208.049829:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff99420693e000 00000800:00000100:0.0:1775768208.049832:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000100:00000001:2.0:1775768208.049834:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000010:3.0:1775768208.049834:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210dcf200. 00000800:00000010:0.0:1775768208.049835:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000020:00000040:2.0:1775768208.049838:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000020:00000010:3.0:1775768208.049838:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207981e40. 00000400:00000200:0.0:1775768208.049841:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000008:00000010:3.0:1775768208.049842:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994208b50000. 00000100:00080000:2.0:1775768208.049843:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775768208.049846:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:3.0:1775768208.049847:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994218000300. 00000400:00000200:0.0:1775768208.049850:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 02000000:00000001:2.0:1775768208.049851:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00008000:00000010:3.0:1775768208.049851:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994207e08e20. 00000400:00000200:0.0:1775768208.049856:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 02000000:00000001:2.0:1775768208.049856:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000010:3.0:1775768208.049856:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99420541b1f8. 02000000:00000001:2.0:1775768208.049859:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775768208.049860:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000080:00000010:3.0:1775768208.049860:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077c0248. 00000100:00000001:2.0:1775768208.049864:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000020:00000010:3.0:1775768208.049864:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcf000. 00000100:00000001:0.0:1775768208.049866:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000008:00100000:2.0:1775768208.049867:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000010:3.0:1775768208.049868:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942086f8420. 00000100:00000200:0.0:1775768208.049872:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dff100 x1862022294287360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768233 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:2.0:1775768208.049873:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768208.049877:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:3.0:1775768208.049877:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994207e8e400. 00000100:00000010:2.0:1775768208.049896:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfc700. 00008000:00000010:3.0:1775768208.049896:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994209270a90. 00000020:00000040:2.0:1775768208.049901:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000080:00000010:3.0:1775768208.049902:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942090e33c0. 02000000:00000001:2.0:1775768208.049906:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000020:00000010:3.0:1775768208.049906:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcf800. 02000000:00000001:2.0:1775768208.049909:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1775768208.049911:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff99420693e000. 00000100:00000001:2.0:1775768208.049913:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775768208.049916:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000080:00000001:3.0:1775768208.049916:0:13471:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1775768208.049919:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:3.0:1775768208.049920:0:13471:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768208.049924:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768208.049927:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dff100 x1862022294287360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768233 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768208.049934:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768208.049936:0:2399:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000010:2.0:1775768208.049937:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1ec00. 00000100:00000001:0.0:1775768208.049942:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768208.049943:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768208.049948:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768208.049951:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4300134205 00000100:00000010:0.0:1775768208.049957:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942078a3d80. 00000100:00100000:2.0:1775768208.049960:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfc700 x1862022294287616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768208.049964:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775768208.049970:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4300134205 00000100:00000010:0.0:1775768208.049974:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942078a30c0. 00000100:00000001:2.0:1775768208.049974:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768208.049978:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1775768208.049978:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768208.049983:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1775768208.049984:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a176dd1] to pc [ptlrpcd_rcv+-1] req@ffff994207dfc700 x1862022294287616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768208.049985:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768208.049990:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768208.049996:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768208/real 1775768208] req@ffff994207dff100 x1862022294287360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768208.049996:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768208.050001:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768208.050008:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000400:00000001:0.0:1775768208.050013:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1775768208.050013:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775768208.050015:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000001:0.0:1775768208.050016:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768208.050019:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000100:00000001:2.0:1775768208.050021:0:36:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:0.0:1775768208.050022:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768208.050025:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00080000:2.0:1775768208.050028:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5460 00000100:00000001:0.0:1775768208.050030:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768208.050037:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dff100 x1862022294287360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1775768208.050038:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768208.050047:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775768208.050052:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000200:0.0:1775768208.050057:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dff100 x1862022294287360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775768208.050059:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768208.050064:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768208.050070:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768208.050074:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775768208.050077:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:0.0:1775768208.050078:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000040:2.0:1775768208.050078:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:2.0:1775768208.050082:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:0.0:1775768208.050083:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:2.0:1775768208.050084:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000100:00000001:0.0:1775768208.050088:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:2.0:1775768208.050088:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:2.0:1775768208.050091:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:0.0:1775768208.050092:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768208.050096:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00100000:0.0:1775768208.050099:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dff100 x1862022294287360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768208.050099:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768208.050104:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768208.050108:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775768208.050113:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00000001:0.0:1775768208.050115:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1775768208.050117:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775768208.050119:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768208.050121:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dff100 x1862022294287360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768233 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1775768208.050122:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775768208.050126:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768208.050130:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768208.050134:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775768208.050135:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00100000:2.0:1775768208.050136:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000100:00000001:0.0:1775768208.050140:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1775768208.050141:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768208.050143:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768208.050144:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768208.050145:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768208.050148:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000010:2.0:1775768208.050148:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfe300. 00000100:00000001:0.0:1775768208.050151:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000040:2.0:1775768208.050152:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00000100:00080000:0.0:1775768208.050178:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:2.0:1775768208.050178:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775768208.050181:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768208.050183:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00080000:0.0:1775768208.050185:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 02000000:00000001:2.0:1775768208.050186:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1775768208.050190:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:2.0:1775768208.050190:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775768208.050194:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1fc00. 00000100:00000040:0.0:1775768208.050195:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dff100 x1862022294287360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768233 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768208.050200:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768208.050210:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294287360:192.168.202.156@tcp:250:kworker.0 00000100:00100000:2.0:1775768208.050210:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfe300 x1862022294287744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768208.050216:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768208.050217:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768208.050219:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dff100 x1862022294287360/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768233 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768208.050232:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768208.050235:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1775768208.050237:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000010:0.0:1775768208.050238:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220029800. 00000100:00000001:2.0:1775768208.050240:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768208.050242:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768208.050245:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000040:2.0:1775768208.050246:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c661fef] to pc [ptlrpcd_rcv+-1] req@ffff994207dfe300 x1862022294287744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1775768208.050247:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:0.0:1775768208.050252:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768208.050255:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1f400. 00000100:00000001:2.0:1775768208.050260:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768208.050261:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768208.050263:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00080000:2.0:1775768208.050263:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000010:0.0:1775768208.050266:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dff100. 00000100:00080000:2.0:1775768208.050267:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775768208.050270:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768208.050270:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1775768208.050273:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1775768208.050274:0:36:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768208.050277:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768208.050278:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775768208.050280:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5460 00000100:00000001:0.0:1775768208.050282:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768208.050284:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768208.050285:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768208.050289:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775768208.050292:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000400:00000200:2.0:1775768208.050292:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:0.0:1775768208.050295:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00080000:2.0:1775768208.050295:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775768208.050300:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000040:0.0:1775768208.050302:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfdc00 x1862022294287488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768208.050302:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768208.050307:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768208.050311:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775768208.050316:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:0.0:1775768208.050317:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000040:2.0:1775768208.050318:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000100:00000001:0.0:1775768208.050319:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768208.050321:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000001:0.0:1775768208.050322:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:2.0:1775768208.050323:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 02000000:00000001:0.0:1775768208.050324:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1775768208.050326:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 02000000:00000001:0.0:1775768208.050327:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768208.050330:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00100000:0.0:1775768208.050333:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294287488:192.168.202.156@tcp:38:kworker.0 00000100:00000040:2.0:1775768208.050336:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768208.050339:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1775768208.050340:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1775768208.050342:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:2.0:1775768208.050344:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000001:0.0:1775768208.050345:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768208.050347:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 02000000:00000001:0.0:1775768208.050348:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:2.0:1775768208.050351:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 02000000:00000010:0.0:1775768208.050353:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220029800. 00000100:00080000:2.0:1775768208.050354:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 02000000:00000001:0.0:1775768208.050359:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768208.050359:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768208.050362:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775768208.050366:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1775768208.050370:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:2.0:1775768208.050370:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768208.050373:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775768208.050375:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000400:00000010:0.0:1775768208.050376:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000008:00000001:2.0:1775768208.050377:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768208.050380:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775768208.050382:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000010:2.0:1775768208.050383:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfed80. 00000020:00000040:2.0:1775768208.050387:0:36:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00000100:00000200:0.0:1775768208.050388:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294287488, portal 10 02000000:00000001:2.0:1775768208.050389:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1775768208.050392:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1775768208.050392:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768208.050394:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1775768208.050396:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611843584 : -112966097708032 : ffff994207dfdc00) 02000000:00000001:2.0:1775768208.050396:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775768208.050399:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775768208.050402:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1f800. 00000100:00000001:2.0:1775768208.050404:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768208.050407:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfdc00 x1862022294287488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768233 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768208.050411:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfed80 x1862022294287872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768208.050425:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768208.050428:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768208.050431:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775768208.050435:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294287488, offset 0 00000100:00000001:2.0:1775768208.050439:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:0.0:1775768208.050442:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768208.050447:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768208.050450:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768208.050453:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768208.050455:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768208.050457:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768208.050458:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00000001:2.0:1775768208.050458:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768208.050464:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225419:225419:256:4294967295] 192.168.202.156@tcp LPNI seq info [225415:225415:8:4294967295] 00000100:00000040:2.0:1775768208.050468:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb9b9eae] to pc [ptlrpcd_rcv+-1] req@ffff994207dfed80 x1862022294287872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775768208.050471:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768208.050484:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000100:00000001:2.0:1775768208.050484:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768208.050489:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5470) 00000800:00000200:0.0:1775768208.050490:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768208.050495:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775768208.050498:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768208.050501:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775768208.050504:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768208.050507:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1775768208.050510:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768208.050511:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775768208.050514:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768208.050516:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768208.050518:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775768208.050519:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775768208.050521:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768208.050524:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfdc00 x1862022294287488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768233 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768208.050530:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768208.050532:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfdc00 x1862022294287488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768233 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768208.050540:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768208.050541:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768208.050544:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768208.050546:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:0.0:1775768208.050548:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768208.050550:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300132883 00000100:00000001:0.0:1775768208.050552:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300134205 00000100:00000001:0.0:1775768208.050553:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300132883 00000100:00000001:0.0:1775768208.050555:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300134205 00000100:00000001:0.0:1775768208.050556:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768208.050558:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768208.050559:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768208.050561:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768208/real 1775768208] req@ffff994207dfdc00 x1862022294287488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768208.050566:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768208.050567:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768208.050568:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775768208.050569:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768208.050570:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768208.050571:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768208.050573:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfdc00 x1862022294287488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768208.050578:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfdc00 x1862022294287488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768208.050582:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768208.050583:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768208.050584:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768208.050585:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768208.050587:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfdc00 x1862022294287488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768208.050592:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768208.050594:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfdc00 x1862022294287488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768233 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768208.050599:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768208.050600:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768208.050601:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768208.050602:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768208.050603:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768208.050605:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768208.050607:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768208.050609:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768208.050610:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768208.050612:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfdc00 x1862022294287488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768233 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768208.050617:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294287488:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768208.050619:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768208.050621:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfc700 x1862022294287616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768208.050626:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768208.050627:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768208.050628:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768208.050629:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768208.050630:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768208.050632:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294287616:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768208.050634:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768208.050635:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768208.050637:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768208.050638:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768208.050640:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002a800. 02000000:00000001:0.0:1775768208.050642:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768208.050644:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768208.050646:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775768208.050648:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768208.050649:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294287616, portal 10 00000100:00000001:0.0:1775768208.050651:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768208.050652:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611838208 : -112966097713408 : ffff994207dfc700) 00000100:00000040:0.0:1775768208.050655:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfc700 x1862022294287616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768233 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768208.050659:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768208.050661:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768208.050662:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775768208.050664:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294287616, offset 0 00000400:00000200:0.0:1775768208.050666:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768208.050668:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768208.050670:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768208.050672:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768208.050673:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768208.050675:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768208.050676:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768208.050679:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225420:225420:256:4294967295] 192.168.202.156@tcp LPNI seq info [225416:225416:8:4294967295] 00000400:00000200:0.0:1775768208.050681:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768208.050686:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768208.050689:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768208.050691:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775768208.050693:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768208.050695:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775768208.050698:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768208.050700:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1775768208.050702:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768208.050704:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775768208.050706:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768208.050708:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768208.050710:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775768208.050711:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775768208.050712:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768208.050714:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfc700 x1862022294287616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768233 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768208.050719:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768208.050721:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfc700 x1862022294287616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768233 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768208.050725:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768208.050726:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768208.050728:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768208.050729:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:0.0:1775768208.050731:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768208.050733:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300132875 00000100:00000001:0.0:1775768208.050735:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300134205 00000100:00000001:0.0:1775768208.050737:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300132875 00000100:00000001:0.0:1775768208.050740:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300134205 00000100:00000001:0.0:1775768208.050742:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768208.050744:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768208.050746:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768208.050749:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768208/real 1775768208] req@ffff994207dfc700 x1862022294287616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768208.050757:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768208.050759:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768208.050760:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775768208.050763:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768208.050764:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768208.050767:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768208.050770:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfc700 x1862022294287616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768208.050778:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfc700 x1862022294287616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768208.050786:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768208.050787:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768208.050789:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768208.050791:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768208.050794:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfc700 x1862022294287616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768208.050801:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768208.050805:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfc700 x1862022294287616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768233 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768208.050811:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768208.050812:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768208.050814:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768208.050814:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768208.050815:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768208.050817:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768208.050819:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768208.050821:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768208.050822:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768208.050824:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfc700 x1862022294287616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768233 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768208.050830:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294287616:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768208.050832:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768208.050835:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfe300 x1862022294287744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768208.050839:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768208.050840:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768208.050842:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768208.050843:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768208.050844:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768208.050846:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294287744:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768208.050848:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768208.050849:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768208.050850:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768208.050852:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768208.050854:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220028400. 02000000:00000001:0.0:1775768208.050856:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768208.050858:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768208.050860:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775768208.050862:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768208.050864:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294287744, portal 4 00000100:00000001:0.0:1775768208.050865:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768208.050866:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611845376 : -112966097706240 : ffff994207dfe300) 00000100:00000040:0.0:1775768208.050870:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfe300 x1862022294287744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768233 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768208.050875:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768208.050876:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768208.050877:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775768208.050879:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294287744, offset 0 00000400:00000200:0.0:1775768208.050896:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768208.050899:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768208.050902:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768208.050904:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768208.050906:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768208.050907:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768208.050909:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768208.050911:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225421:225421:256:4294967295] 192.168.202.156@tcp LPNI seq info [225417:225417:8:4294967295] 00000400:00000200:0.0:1775768208.050914:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768208.050919:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768208.050922:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768208.050924:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775768208.050927:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768208.050929:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775768208.050931:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768208.050933:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1775768208.050936:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768208.050937:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775768208.050939:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768208.050941:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768208.050943:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775768208.050944:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775768208.050946:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768208.050948:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfe300 x1862022294287744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768233 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768208.050953:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768208.050955:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfe300 x1862022294287744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768233 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768208.050960:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768208.050961:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768208.050962:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768208.050965:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:0.0:1775768208.050967:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768208.050968:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300132898 00000100:00000001:0.0:1775768208.050969:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300134205 00000100:00000001:0.0:1775768208.050971:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300132898 00000100:00000001:0.0:1775768208.050972:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300134205 00000100:00000001:0.0:1775768208.050974:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768208.050975:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768208.050977:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768208.050979:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768208/real 1775768208] req@ffff994207dfe300 x1862022294287744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768208.050984:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768208.050985:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768208.050986:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775768208.050987:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768208.050988:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768208.050989:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768208.050991:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfe300 x1862022294287744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768208.050997:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfe300 x1862022294287744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768208.051001:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768208.051002:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768208.051003:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768208.051003:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768208.051005:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfe300 x1862022294287744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768208.051009:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768208.051012:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfe300 x1862022294287744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768233 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768208.051016:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768208.051017:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768208.051018:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768208.051019:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768208.051020:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768208.051022:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768208.051024:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768208.051026:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768208.051028:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768208.051032:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfe300 x1862022294287744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768233 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768208.051040:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294287744:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768208.051044:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768208.051046:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768208.051049:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfdc00 x1862022294287488/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768233 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768208.051055:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768208.051057:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768208.051059:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220029800. 02000000:00000001:0.0:1775768208.051062:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768208.051063:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768208.051064:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:0.0:1775768208.051067:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768208.051069:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1f000. 02000000:00000001:0.0:1775768208.051071:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768208.051073:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768208.051074:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfdc00. 00000100:00000001:0.0:1775768208.051077:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768208.051078:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768208.051080:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768208.051084:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfc700 x1862022294287616/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768233 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768208.051091:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768208.051092:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768208.051093:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002a800. 02000000:00000001:0.0:1775768208.051095:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768208.051096:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768208.051096:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:0.0:1775768208.051098:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768208.051099:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1ec00. 02000000:00000001:0.0:1775768208.051101:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768208.051102:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768208.051103:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfc700. 00000100:00000001:0.0:1775768208.051104:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768208.051105:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768208.051106:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768208.051108:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfe300 x1862022294287744/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768233 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768208.051112:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768208.051113:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768208.051114:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220028400. 02000000:00000001:0.0:1775768208.051115:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768208.051116:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768208.051117:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:0.0:1775768208.051118:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768208.051119:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1fc00. 02000000:00000001:0.0:1775768208.051120:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768208.051121:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768208.051121:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfe300. 00000100:00000001:0.0:1775768208.051123:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768208.051123:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768208.051124:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768208.051127:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768208.051128:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768208.051129:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775768208.051130:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768208.051131:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768208.051133:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfed80 x1862022294287872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768208.051137:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768208.051138:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768208.051139:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768208.051141:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768208.051142:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768208.051144:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294287872:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768208.051146:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768208.051148:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768208.051149:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768208.051150:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768208.051152:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220028400. 02000000:00000001:0.0:1775768208.051169:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768208.051172:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768208.051174:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775768208.051176:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768208.051179:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294287872, portal 4 00000100:00000001:0.0:1775768208.051180:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768208.051181:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611848064 : -112966097703552 : ffff994207dfed80) 00000100:00000040:0.0:1775768208.051185:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfed80 x1862022294287872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768233 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768208.051190:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768208.051191:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768208.051193:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775768208.051194:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294287872, offset 0 00000400:00000200:0.0:1775768208.051196:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768208.051199:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768208.051201:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768208.051203:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768208.051205:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768208.051206:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768208.051208:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768208.051210:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225422:225422:256:4294967295] 192.168.202.156@tcp LPNI seq info [225418:225418:8:4294967295] 00000400:00000200:0.0:1775768208.051213:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768208.051217:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768208.051219:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768208.051222:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775768208.051224:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768208.051226:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775768208.051229:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768208.051231:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1775768208.051233:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768208.051235:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775768208.051237:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768208.051239:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768208.051241:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775768208.051242:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775768208.051244:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768208.051246:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfed80 x1862022294287872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768233 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768208.051251:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768208.051253:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfed80 x1862022294287872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768233 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768208.051257:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768208.051258:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768208.051260:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768208.051262:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:0.0:1775768208.051264:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768208.051265:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300132902 00000100:00000001:0.0:1775768208.051267:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300134205 00000100:00000001:0.0:1775768208.051268:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300132902 00000100:00000001:0.0:1775768208.051270:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300134205 00000100:00000001:0.0:1775768208.051271:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768208.051272:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768208.051273:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768208.051275:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768208/real 1775768208] req@ffff994207dfed80 x1862022294287872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768208.051280:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768208.051281:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768208.051282:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775768208.051283:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768208.051284:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768208.051286:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768208.051288:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfed80 x1862022294287872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768208.051293:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfed80 x1862022294287872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768208.051296:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768208.051297:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768208.051298:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768208.051299:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768208.051301:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfed80 x1862022294287872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768208.051305:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768208.051307:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfed80 x1862022294287872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768233 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768208.051311:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768208.051313:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768208.051313:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768208.051314:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768208.051315:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768208.051318:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768208.051320:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768208.051322:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768208.051324:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768208.051326:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfed80 x1862022294287872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768233 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768208.051331:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294287872:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768208.051334:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768208.051335:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768208.051337:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfed80 x1862022294287872/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768233 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768208.051341:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768208.051342:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768208.051343:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220028400. 02000000:00000001:0.0:1775768208.051344:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768208.051345:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768208.051346:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:0.0:1775768208.051347:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768208.051348:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1f800. 02000000:00000001:0.0:1775768208.051349:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768208.051350:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768208.051351:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfed80. 00000100:00000001:0.0:1775768208.051353:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768208.051354:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768208.051355:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768208.051358:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768208.051360:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768208.051361:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775768208.051363:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768208.113265:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768208.113270:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768208.113275:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768208.113277:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768208.113279:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768208.113280:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768208.113283:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768208.113283:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768208.113285:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768208.113286:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768208.113358:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768208.113363:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768208.113366:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768208.177269:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768208.177275:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768208.177277:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768208.177282:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768208.177285:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768208.177288:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768208.177290:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768208.177291:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768208.177293:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768209.073262:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768209.073268:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768209.073269:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768209.073275:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:0.0:1775768209.073279:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768209.073280:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768209.073283:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768209.073286:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768209.073287:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768209.073290:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768209.073290:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768209.073291:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768209.073296:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768209.073297:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768209.073300:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768209.073304:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:1.0:1775768209.073306:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768209.073308:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768209.073312:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768209.073313:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768209.073317:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768209.073317:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768209.073320:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768209.073323:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768209.073324:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768209.073327:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768209.073330:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768209.137251:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768209.137257:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768209.137261:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768209.137262:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768209.137265:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768209.137266:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768209.137268:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768209.137268:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768209.137271:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768209.137272:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768209.137330:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768209.137334:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768209.137337:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768209.201259:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768209.201265:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768209.201267:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768209.201271:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768209.201275:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768209.201278:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768209.201280:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768209.201282:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768209.201284:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768209.393259:0:15036:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4300134205, at 4300134208 00000100:00000040:1.0:1775768209.393259:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4300134205, at 4300134208 00000100:00000010:0.0:1775768209.393272:0:15036:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3d80. 00000100:00000010:1.0:1775768209.393274:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a30c0. 00000100:00000001:0.0:1775768210.097257:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768210.097257:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768210.097257:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768210.097263:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768210.097264:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768210.097264:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768210.097267:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768210.097267:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768210.097268:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768210.097275:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:1.0:1775768210.097275:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:2.0:1775768210.097276:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768210.097281:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768210.097281:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768210.097282:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768210.097285:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768210.097285:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768210.097286:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768210.097290:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768210.097293:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768210.097295:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768210.097296:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768210.097299:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768210.097302:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768210.097303:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768210.097306:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768210.097309:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768210.161263:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768210.161268:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768210.161273:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768210.161274:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768210.161276:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768210.161278:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768210.161280:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768210.161281:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768210.161284:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768210.161285:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768210.161344:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768210.161348:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768210.161352:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768210.225281:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768210.225286:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768210.225288:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768210.225292:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768210.225295:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768210.225298:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768210.225300:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768210.225302:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768210.225304:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768211.121219:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768211.121219:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768211.121226:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768211.121226:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768211.121227:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768211.121227:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768211.121231:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768211.121232:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:2.0:1775768211.121232:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775768211.121236:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768211.121237:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1775768211.121238:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:0.0:1775768211.121241:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768211.121242:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768211.121242:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768211.121246:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768211.121246:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768211.121247:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768211.121250:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768211.121250:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768211.121254:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768211.121258:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768211.121261:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768211.121263:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768211.121264:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768211.121267:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768211.121270:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768211.185228:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768211.185234:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768211.185239:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768211.185241:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768211.185244:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768211.185245:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768211.185248:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768211.185249:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768211.185253:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768211.185254:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768211.185288:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768211.185291:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768211.185293:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768211.249238:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768211.249244:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768211.249246:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768211.249251:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768211.249254:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768211.249258:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768211.249260:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768211.249263:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768211.249266:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768212.145256:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768212.145256:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768212.145256:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768212.145262:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768212.145263:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768212.145263:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768212.145265:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768212.145267:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768212.145267:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768212.145273:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:0.0:1775768212.145274:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:2.0:1775768212.145274:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768212.145278:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768212.145279:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768212.145280:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768212.145284:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768212.145284:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768212.145287:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768212.145288:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768212.145299:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768212.145302:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768212.145302:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768212.145305:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768212.145307:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768212.145308:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768212.145310:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768212.145312:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768212.209261:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768212.209268:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768212.209270:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768212.209273:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768212.209278:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768212.209280:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768212.209282:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768212.209284:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768212.209288:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768212.209289:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768212.209301:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768212.209303:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768212.209305:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768212.209307:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768212.209310:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768212.209312:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768212.209372:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768212.209375:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768212.209377:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768212.273258:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768212.273265:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768212.273267:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768212.273272:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768212.273276:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768212.273279:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768212.273282:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768212.273284:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768212.273286:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768213.169251:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768213.169252:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768213.169255:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768213.169258:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:2.0:1775768213.169258:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1775768213.169260:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768213.169261:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768213.169264:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768213.169264:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768213.169267:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768213.169267:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:0.0:1775768213.169268:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775768213.169269:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768213.169271:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:1.0:1775768213.169273:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768213.169274:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768213.169275:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1775768213.169276:0:36:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768213.169277:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768213.169279:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768213.169280:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768213.169281:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768213.169281:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:3.0:1775768213.169285:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768213.169288:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1775768213.169291:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5465 00000100:00000001:3.0:1775768213.169292:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768213.169293:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768213.169296:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768213.169300:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768213.169301:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000400:00000200:2.0:1775768213.169303:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768213.169305:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768213.169309:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768213.169312:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775768213.169316:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775768213.169320:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768213.169324:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768213.169327:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768213.169329:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775768213.169333:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000080:00000001:3.0:1775768213.169333:0:13471:0:(rw.c:1481:ll_writepages()) Process entered 00000020:00000040:2.0:1775768213.169334:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:2.0:1775768213.169337:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000080:00000001:3.0:1775768213.169337:0:13471:0:(file.c:5250:cl_sync_file_range()) Process entered 00000020:00000040:2.0:1775768213.169339:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:2.0:1775768213.169341:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:3.0:1775768213.169341:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:3.0:1775768213.169350:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff99420693e000. 00000020:00000001:2.0:1775768213.169351:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:2.0:1775768213.169354:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768213.169356:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000010:3.0:1775768213.169356:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcf000. 00000100:00000040:2.0:1775768213.169359:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768213.169361:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000010:3.0:1775768213.169362:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210dcf800. 00000020:00000040:2.0:1775768213.169363:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:2.0:1775768213.169368:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775768213.169369:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1775768213.169369:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207981e40. 02000000:00000001:2.0:1775768213.169373:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775768213.169375:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768213.169377:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:3.0:1775768213.169378:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994208b50000. 00000100:00000001:2.0:1775768213.169380:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775768213.169382:0:36:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:3.0:1775768213.169384:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994218000300. 00000100:00000010:2.0:1775768213.169387:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfce00. 00000020:00000040:2.0:1775768213.169390:0:36:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 00008000:00000010:3.0:1775768213.169391:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994207e08e20. 02000000:00000001:2.0:1775768213.169393:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775768213.169395:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768213.169398:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000080:00000010:3.0:1775768213.169398:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99420541b1f8. 02000000:00000001:2.0:1775768213.169401:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775768213.169404:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:3.0:1775768213.169404:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077c0248. 00000020:00000010:3.0:1775768213.169410:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcf200. 02000000:00000010:2.0:1775768213.169412:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1f800. 00000008:00000010:3.0:1775768213.169415:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942086f8420. 00000100:00000001:2.0:1775768213.169418:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:3.0:1775768213.169420:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994207e8e400. 00008000:00000010:3.0:1775768213.169425:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994209270a90. 00000100:00100000:2.0:1775768213.169426:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfce00 x1862022294288000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000080:00000010:3.0:1775768213.169430:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942090e33c0. 00000020:00000001:3.0:1775768213.169435:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107590086664 : -112966119464952 : ffff99420693e008) 00000100:00000001:2.0:1775768213.169436:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000020:00001000:3.0:1775768213.169439:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff99420693e000 00000100:00000001:2.0:1775768213.169440:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768213.169444:0:13471:0:(cl_object.c:151:cl_object_top()) ffff9942078a76c0 -> ffff9942078a76c0 00000100:00000040:2.0:1775768213.169445:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e5a7debe] to pc [ptlrpcd_rcv+-1] req@ffff994207dfce00 x1862022294288000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775768213.169449:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:3.0:1775768213.169453:0:13471:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:3.0:1775768213.169456:0:13471:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:3.0:1775768213.169459:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:3.0:1775768213.169461:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00000001:2.0:1775768213.169462:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:3.0:1775768213.169463:0:13471:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768213.169467:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00020000:00000002:3.0:1775768213.169469:0:13471:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017a:0x0]io ffff9942077c02a8 type 6 ignore/verify layout 1/0 00000100:00080000:2.0:1775768213.169470:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775768213.169472:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775768213.169474:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775768213.169477:0:36:0:(import.c:484:import_select_connection()) Process entered 00020000:00000002:3.0:1775768213.169477:0:13471:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207c1f9b0) owner 0000000000000000/ffff994220080000 00000100:00080000:2.0:1775768213.169480:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5465 00020000:00000001:3.0:1775768213.169482:0:13471:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00000400:00000200:2.0:1775768213.169485:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:3.0:1775768213.169486:0:13471:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00000100:00000001:0.0:1775768213.169489:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:2.0:1775768213.169489:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00020000:00000001:3.0:1775768213.169490:0:13471:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00000100:00000001:0.0:1775768213.169492:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1775768213.169494:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00020000:00000001:3.0:1775768213.169494:0:13471:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768213.169495:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00020000:00000001:3.0:1775768213.169499:0:13471:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1775768213.169503:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000040:0.0:1775768213.169504:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfce00 x1862022294288000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:3.0:1775768213.169504:0:13471:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00000100:00000001:2.0:1775768213.169508:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00020000:00000001:3.0:1775768213.169508:0:13471:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768213.169512:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000001:3.0:1775768213.169514:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000100:00000001:2.0:1775768213.169516:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775768213.169517:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:2.0:1775768213.169521:0:36:0:(genops.c:811:class_conn2export()) Process entered 00020000:00000001:3.0:1775768213.169521:0:13471:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00000020:00000040:2.0:1775768213.169525:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00020000:00000002:3.0:1775768213.169525:0:13471:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207c1f9b0) owner 0000000000000000/ffff994220080000 00000100:00000001:0.0:1775768213.169527:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000001:2.0:1775768213.169528:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:0.0:1775768213.169530:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768213.169531:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775768213.169532:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 02000000:00000001:0.0:1775768213.169536:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:2.0:1775768213.169536:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:3.0:1775768213.169537:0:13471:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:2.0:1775768213.169538:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 02000000:00000001:0.0:1775768213.169541:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1775768213.169541:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000001:3.0:1775768213.169541:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000100:00000001:2.0:1775768213.169545:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00020000:00000001:3.0:1775768213.169545:0:13471:0:(lov_io.c:836:lov_io_iter_init()) Process entered 02000000:00000001:0.0:1775768213.169546:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768213.169548:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00020000:00200000:3.0:1775768213.169550:0:13471:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00000001:2.0:1775768213.169551:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00100000:0.0:1775768213.169554:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294288000:192.168.202.156@tcp:250:kworker.0 00000020:00000040:2.0:1775768213.169555:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00020000:00000002:3.0:1775768213.169557:0:13471:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00000100:00080000:2.0:1775768213.169560:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775768213.169564:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768213.169565:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00020000:00000001:3.0:1775768213.169565:0:13471:0:(lov_io.c:155:lov_sub_get()) Process entered 02000000:00000001:2.0:1775768213.169566:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00020000:00000001:3.0:1775768213.169570:0:13471:0:(lov_io.c:104:lov_io_sub_init()) Process entered 02000000:00000001:0.0:1775768213.169571:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768213.169572:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1775768213.169572:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 02000000:00000001:0.0:1775768213.169575:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768213.169575:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1775768213.169576:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff99420693ecf0. 00000100:00000001:2.0:1775768213.169580:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:0.0:1775768213.169581:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000010:3.0:1775768213.169581:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dce800. 00000008:00100000:2.0:1775768213.169583:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775768213.169587:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768213.169588:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220028400. 00000100:00000001:2.0:1775768213.169588:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1775768213.169589:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210dcf600. 00000100:00000010:2.0:1775768213.169591:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dff480. 02000000:00000001:0.0:1775768213.169593:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775768213.169595:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000010:3.0:1775768213.169596:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207981900. 02000000:00000001:2.0:1775768213.169597:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000200:0.0:1775768213.169600:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:2.0:1775768213.169600:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768213.169602:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000008:00000010:3.0:1775768213.169602:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994208b51650. 00000400:00000010:0.0:1775768213.169604:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 02000000:00000001:2.0:1775768213.169604:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775768213.169607:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:3.0:1775768213.169609:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994218000000. 02000000:00000010:2.0:1775768213.169611:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1fc00. 00000100:00000001:2.0:1775768213.169614:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:3.0:1775768213.169614:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994207e0ade8. 00000400:00000010:0.0:1775768213.169619:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00100000:2.0:1775768213.169620:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dff480 x1862022294288128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000080:00000010:3.0:1775768213.169621:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99420541b0a8. 00000080:00000010:3.0:1775768213.169626:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077c1da8. 00000100:00000200:0.0:1775768213.169629:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294288000, portal 25 00000100:00000001:2.0:1775768213.169631:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768213.169633:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000020:00000010:3.0:1775768213.169633:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dce400. 00000100:00000001:2.0:1775768213.169634:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768213.169637:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611840000 : -112966097711616 : ffff994207dfce00) 00000008:00000010:3.0:1775768213.169639:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942086f8160. 00000100:00000040:2.0:1775768213.169640:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093a15639] to pc [ptlrpcd_rcv+-1] req@ffff994207dff480 x1862022294288128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000010:3.0:1775768213.169645:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994207e8ed00. 00000100:00000040:0.0:1775768213.169647:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfce00 x1862022294288000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768238 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00008000:00000010:3.0:1775768213.169650:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994209270ef0. 00000080:00000010:3.0:1775768213.169656:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942090e3960. 00000100:00000001:2.0:1775768213.169657:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768213.169662:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000001:3.0:1775768213.169662:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107590089976 : -112966119461640 : ffff99420693ecf8) 00000100:00000001:0.0:1775768213.169666:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1775768213.169666:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000020:00001000:3.0:1775768213.169668:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff99420693ecf0 00000100:00000001:2.0:1775768213.169669:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775768213.169671:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:0.0:1775768213.169672:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000020:00000001:3.0:1775768213.169674:0:13471:0:(cl_object.c:151:cl_object_top()) ffff9942078a7788 -> ffff9942078a76c0 00000100:00000001:2.0:1775768213.169675:0:36:0:(import.c:484:import_select_connection()) Process entered 00000400:00000010:0.0:1775768213.169676:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000020:00000001:3.0:1775768213.169678:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000100:00080000:2.0:1775768213.169679:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5465 00000100:00000200:0.0:1775768213.169680:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294288000, offset 0 00000020:00000001:3.0:1775768213.169682:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:3.0:1775768213.169684:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000400:00000200:2.0:1775768213.169685:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:3.0:1775768213.169687:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768213.169689:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00020000:00000001:3.0:1775768213.169691:0:13471:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768213.169694:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00020000:00000001:3.0:1775768213.169694:0:13471:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107612435560 : -112966097116056 : ffff994207e8e468) 00000400:00000200:0.0:1775768213.169697:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000020:00000001:3.0:1775768213.169699:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000100:00000040:2.0:1775768213.169701:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000001:3.0:1775768213.169702:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768213.169706:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00020000:00000001:3.0:1775768213.169707:0:13471:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768213.169710:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000040:2.0:1775768213.169712:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000001:3.0:1775768213.169712:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768213.169717:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775768213.169717:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00000400:00000200:0.0:1775768213.169719:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00020000:00000001:3.0:1775768213.169720:0:13471:0:(lov_io.c:1108:lov_io_lock()) Process entered 00000020:00000001:2.0:1775768213.169722:0:36:0:(genops.c:811:class_conn2export()) Process entered 00020000:00000001:3.0:1775768213.169724:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000040:2.0:1775768213.169725:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:3.0:1775768213.169726:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:2.0:1775768213.169728:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:0.0:1775768213.169729:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:3.0:1775768213.169730:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000040:2.0:1775768213.169731:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:3.0:1775768213.169733:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768213.169735:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000020:00000001:2.0:1775768213.169736:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:3.0:1775768213.169737:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768213.169742:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000400:00000200:0.0:1775768213.169743:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00020000:00000001:3.0:1775768213.169743:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768213.169748:0:13471:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768213.169750:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000400:00000200:0.0:1775768213.169751:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00000040:2.0:1775768213.169752:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000001:3.0:1775768213.169753:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768213.169756:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:3.0:1775768213.169757:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768213.169758:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225423:225423:256:4294967295] 192.168.202.156@tcp LPNI seq info [225419:225419:8:4294967295] 00000100:00000040:2.0:1775768213.169760:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000001:3.0:1775768213.169762:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00000100:00000001:2.0:1775768213.169765:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00020000:00000001:3.0:1775768213.169766:0:13471:0:(lov_io.c:1114:lov_io_start()) Process entered 00000400:00000200:0.0:1775768213.169769:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:3.0:1775768213.169769:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000040:2.0:1775768213.169770:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000020:00000001:3.0:1775768213.169771:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00000100:00080000:2.0:1775768213.169774:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000008:00000001:3.0:1775768213.169774:0:13471:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000100:00000001:2.0:1775768213.169778:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1775768213.169779:0:13471:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 02000000:00000001:2.0:1775768213.169780:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775768213.169784:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1775768213.169786:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 02000000:00000001:2.0:1775768213.169787:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768213.169791:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775768213.169793:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775768213.169795:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000800:00000200:0.0:1775768213.169796:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000008:00000001:2.0:1775768213.169797:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000008:00000020:3.0:1775768213.169799:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000100:00000001:2.0:1775768213.169800:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1775768213.169801:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:0.0:1775768213.169802:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000100:00000010:2.0:1775768213.169804:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfed80. 00000008:00000001:3.0:1775768213.169807:0:13471:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000020:00000040:2.0:1775768213.169808:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000008:00000001:3.0:1775768213.169809:0:13471:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000800:00000200:0.0:1775768213.169811:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 02000000:00000001:2.0:1775768213.169813:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000008:00000001:3.0:1775768213.169813:0:13471:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107884691392 : -112965824860224 : ffff994218232fc0) 00000800:00000200:0.0:1775768213.169815:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 02000000:00000001:2.0:1775768213.169816:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000008:00000020:3.0:1775768213.169818:0:13471:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994218232fc0 ready 0|-|- wr 1|-|+ rd 0|- 10 in flight 00000100:00000001:2.0:1775768213.169820:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775768213.169823:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:0.1:1775768213.169823:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000008:00000001:3.0:1775768213.169825:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 02000000:00000001:2.0:1775768213.169827:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000020:3.0:1775768213.169828:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000800:00000200:0.1:1775768213.169828:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000008:00000001:3.0:1775768213.169830:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:2.0:1775768213.169843:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1ec00. 00000008:00000001:3.0:1775768213.169845:0:13471:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000100:00000001:2.0:1775768213.169849:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000008:00000020:3.0:1775768213.169849:0:13471:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994210675880 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000008:00000001:3.0:1775768213.169856:0:13471:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768213.169858:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfed80 x1862022294288256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768213.169859:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000008:00000001:3.0:1775768213.169861:0:13471:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1775768213.169864:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000020:00000001:3.0:1775768213.169866:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768213.169871:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768213.169872:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00020000:00000001:3.0:1775768213.169875:0:13471:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768213.169876:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775768213.169879:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768213.169879:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768213.169882:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00000100:00000040:2.0:1775768213.169883:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb9b9eae] to pc [ptlrpcd_rcv+-1] req@ffff994207dfed80 x1862022294288256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:3.0:1775768213.169885:0:13471:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00020000:00000001:3.0:1775768213.169889:0:13471:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000400:00000200:0.0:1775768213.169891:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000020:00000001:3.0:1775768213.169892:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000001:3.0:1775768213.169895:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00000100:00000001:2.0:1775768213.169896:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768213.169898:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000100:00080000:2.0:1775768213.169899:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00020000:00000001:3.0:1775768213.169899:0:13471:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775768213.169902:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00020000:00000001:3.0:1775768213.169903:0:13471:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000100:00080000:2.0:1775768213.169907:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000020:00000001:3.0:1775768213.169907:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00000100:00000001:0.0:1775768213.169908:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:3.0:1775768213.169910:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000100:00000001:2.0:1775768213.169911:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000200:0.0:1775768213.169914:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfce00 x1862022294288000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768238 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768213.169915:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00020000:00000001:3.0:1775768213.169915:0:13471:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00020000:00000001:3.0:1775768213.169918:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000001:2.0:1775768213.169920:0:36:0:(import.c:484:import_select_connection()) Process entered 00000020:00000001:3.0:1775768213.169921:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000100:00080000:2.0:1775768213.169924:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5465 00000020:00000001:3.0:1775768213.169925:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:3.0:1775768213.169928:0:13471:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768213.169932:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:3.0:1775768213.169933:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768213.169937:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00020000:00000001:3.0:1775768213.169937:0:13471:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000400:00000200:2.0:1775768213.169939:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000020:00000001:3.0:1775768213.169940:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000100:00080000:2.0:1775768213.169942:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000020:00000001:3.0:1775768213.169944:0:13471:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000100:00000040:0.0:1775768213.169945:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfce00 x1862022294288000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768238 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775768213.169947:0:13471:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000100:00000040:2.0:1775768213.169949:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000001:3.0:1775768213.169951:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000100:00000001:2.0:1775768213.169954:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00020000:00000001:3.0:1775768213.169954:0:13471:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00020000:00000001:3.0:1775768213.169957:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000040:2.0:1775768213.169960:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000001:3.0:1775768213.169960:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000100:00000001:0.0:1775768213.169963:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768213.169963:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000100:00000001:2.0:1775768213.169964:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00020000:00000001:3.0:1775768213.169966:0:13471:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768213.169967:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:2.0:1775768213.169969:0:36:0:(genops.c:811:class_conn2export()) Process entered 00020000:00000001:3.0:1775768213.169971:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775768213.169972:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00020000:00000001:3.0:1775768213.169974:0:13471:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000020:00000001:2.0:1775768213.169975:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:0.0:1775768213.169976:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768213.169977:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000020:00000040:2.0:1775768213.169978:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:3.0:1775768213.169980:0:13471:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768213.169983:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000400:00000200:0.0:1775768213.169984:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:3.0:1775768213.169984:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00000100:00000001:0.0:1775768213.169988:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4300139325 00000020:00000001:2.0:1775768213.169988:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:2.0:1775768213.169992:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000010:0.0:1775768213.169993:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942078a3d80. 00020000:00000001:3.0:1775768213.169993:0:13471:0:(lov_io.c:675:lov_io_fini()) Process entered 00000100:00000001:2.0:1775768213.169995:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00020000:00000001:3.0:1775768213.169997:0:13471:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000100:00000040:2.0:1775768213.169999:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000001:3.0:1775768213.170000:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00000100:00000001:2.0:1775768213.170001:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:0.0:1775768213.170003:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000020:00000001:3.0:1775768213.170003:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00000040:2.0:1775768213.170005:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000020:00001000:3.0:1775768213.170007:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff99420693ecf0 00000100:00080000:2.0:1775768213.170008:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768213.170010:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4300139325 00000100:00000001:2.0:1775768213.170012:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1775768213.170013:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210dcf600. 00000100:00000010:0.0:1775768213.170015:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942078a3600. 02000000:00000001:2.0:1775768213.170015:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775768213.170019:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1775768213.170019:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207981900. 00000100:00000001:0.0:1775768213.170020:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 02000000:00000001:2.0:1775768213.170021:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768213.170025:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775768213.170027:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00100000:2.0:1775768213.170027:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000010:3.0:1775768213.170027:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994208b51650. 00000008:00000001:2.0:1775768213.170031:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768213.170033:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768213.170033:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:3.0:1775768213.170034:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994218000000. 00000100:00000010:2.0:1775768213.170037:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfe300. 00000100:00000001:0.0:1775768213.170038:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00008000:00000010:3.0:1775768213.170039:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994207e0ade8. 00000020:00000040:2.0:1775768213.170042:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00000080:00000010:3.0:1775768213.170045:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99420541b0a8. 00000100:00100000:0.0:1775768213.170046:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768213/real 1775768213] req@ffff994207dfce00 x1862022294288000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1775768213.170047:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775768213.170050:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:3.0:1775768213.170051:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077c1da8. 00000100:00000001:2.0:1775768213.170055:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775768213.170059:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000020:00000010:3.0:1775768213.170059:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dce800. 02000000:00000001:2.0:1775768213.170062:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1775768213.170063:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000008:00000010:3.0:1775768213.170064:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942086f8160. 00000400:00000001:0.0:1775768213.170066:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768213.170069:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 02000000:00000010:2.0:1775768213.170069:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1f000. 00020000:00000010:3.0:1775768213.170070:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994207e8ed00. 00000400:00000200:0.0:1775768213.170076:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000100:00000001:2.0:1775768213.170076:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:3.0:1775768213.170077:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994209270ef0. 00000400:00000010:0.0:1775768213.170080:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000080:00000010:3.0:1775768213.170083:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942090e3960. 00000100:00100000:2.0:1775768213.170087:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfe300 x1862022294288384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000010:3.0:1775768213.170088:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dce400. 00000100:00000001:0.0:1775768213.170093:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000010:3.0:1775768213.170093:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff99420693ecf0. 00020000:00000001:3.0:1775768213.170099:0:13471:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00000100:00000200:0.0:1775768213.170100:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfce00 x1862022294288000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:3.0:1775768213.170102:0:13471:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:3.0:1775768213.170105:0:13471:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000100:00000001:2.0:1775768213.170109:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000080:00200000:3.0:1775768213.170109:0:13471:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00000001:2.0:1775768213.170114:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:3.0:1775768213.170118:0:13471:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:3.0:1775768213.170121:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000100:00000040:2.0:1775768213.170123:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c661fef] to pc [ptlrpcd_rcv+-1] req@ffff994207dfe300 x1862022294288384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00001000:3.0:1775768213.170126:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff99420693e000 00000020:00000010:3.0:1775768213.170130:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210dcf800. 00000100:00000200:0.0:1775768213.170132:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfce00 x1862022294288000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1775768213.170135:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207981e40. 00000008:00000010:3.0:1775768213.170140:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994208b50000. 00020000:00000010:3.0:1775768213.170145:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994218000300. 00000100:00000001:2.0:1775768213.170147:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:3.0:1775768213.170150:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994207e08e20. 00000100:00080000:2.0:1775768213.170151:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775768213.170152:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768213.170173:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768213.170176:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:2.0:1775768213.170177:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775768213.170179:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768213.170180:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000080:00000010:3.0:1775768213.170182:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99420541b1f8. 00000100:00080000:2.0:1775768213.170183:0:36:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000080:00000010:3.0:1775768213.170188:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077c0248. 00000100:00000001:2.0:1775768213.170189:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00100000:0.0:1775768213.170191:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfce00 x1862022294288000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1775768213.170193:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcf000. 00000100:00080000:2.0:1775768213.170196:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5465 00000008:00000010:3.0:1775768213.170200:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942086f8420. 00020000:00000010:3.0:1775768213.170205:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994207e8e400. 00000400:00000200:2.0:1775768213.170206:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768213.170211:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00008000:00000010:3.0:1775768213.170211:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994209270a90. 00000400:00000200:2.0:1775768213.170213:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775768213.170216:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000080:00000010:3.0:1775768213.170217:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942090e33c0. 00000100:00000040:0.0:1775768213.170220:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfce00 x1862022294288000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768238 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775768213.170222:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000010:3.0:1775768213.170223:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcf200. 00000100:00000001:2.0:1775768213.170226:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000010:3.0:1775768213.170229:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff99420693e000. 00000100:00000040:2.0:1775768213.170231:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768213.170235:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000080:00000001:3.0:1775768213.170235:0:13471:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768213.170236:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:2.0:1775768213.170240:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:0.0:1775768213.170241:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:3.0:1775768213.170241:0:13471:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775768213.170243:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000100:00000001:0.0:1775768213.170244:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768213.170247:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1775768213.170247:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:0.0:1775768213.170250:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000020:00000040:2.0:1775768213.170250:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000080:00000001:3.0:1775768213.170252:0:13471:0:(rw.c:1481:ll_writepages()) Process entered 00000020:00000001:2.0:1775768213.170253:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000001:0.0:1775768213.170254:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000080:00000001:3.0:1775768213.170254:0:13471:0:(file.c:5250:cl_sync_file_range()) Process entered 00000020:00000001:2.0:1775768213.170256:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:3.0:1775768213.170257:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00080000:0.0:1775768213.170261:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000040:2.0:1775768213.170261:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000010:3.0:1775768213.170262:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff99420693e000. 00000100:00000001:2.0:1775768213.170266:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000010:3.0:1775768213.170268:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcf200. 00000100:00080000:0.0:1775768213.170272:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000040:2.0:1775768213.170272:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000010:3.0:1775768213.170274:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210dcf000. 00000100:00000001:2.0:1775768213.170277:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:0.0:1775768213.170279:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000010:3.0:1775768213.170279:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207981e40. 00000020:00000040:2.0:1775768213.170281:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000008:00000010:3.0:1775768213.170285:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994208b50000. 00000100:00080000:2.0:1775768213.170288:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000040:0.0:1775768213.170289:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfce00 x1862022294288000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768238 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000010:3.0:1775768213.170290:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994218000300. 00000100:00000001:2.0:1775768213.170293:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:3.0:1775768213.170294:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994207e08e20. 02000000:00000001:2.0:1775768213.170299:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000080:00000010:3.0:1775768213.170300:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99420541b1f8. 02000000:00000001:2.0:1775768213.170304:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000010:3.0:1775768213.170306:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077c0248. 02000000:00000001:2.0:1775768213.170309:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1775768213.170311:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcf800. 00000100:00100000:0.0:1775768213.170312:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294288000:192.168.202.156@tcp:250:kworker.0 00000100:00000001:2.0:1775768213.170313:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775768213.170316:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000010:3.0:1775768213.170316:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942086f8420. 00000100:00000001:0.0:1775768213.170319:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768213.170321:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000008:00000001:2.0:1775768213.170321:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:3.0:1775768213.170321:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994207e8e400. 00000100:00000001:2.0:1775768213.170325:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768213.170326:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfce00 x1862022294288000/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768238 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00008000:00000010:3.0:1775768213.170326:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994209270a90. 00000100:00000010:2.0:1775768213.170329:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfc700. 00000020:00000040:2.0:1775768213.170333:0:36:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00000080:00000010:3.0:1775768213.170333:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942090e33c0. 00000020:00000001:3.0:1775768213.170338:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107590086664 : -112966119464952 : ffff99420693e008) 02000000:00000001:2.0:1775768213.170339:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775768213.170341:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:3.0:1775768213.170345:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff99420693e000 00000100:00000001:2.0:1775768213.170346:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775768213.170348:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1775768213.170351:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768213.170351:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768213.170351:0:13471:0:(cl_object.c:151:cl_object_top()) ffff9942078a7f58 -> ffff9942078a7f58 02000000:00000001:0.0:1775768213.170355:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000001:3.0:1775768213.170356:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 02000000:00000010:2.0:1775768213.170357:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1f400. 02000000:00000010:0.0:1775768213.170358:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220028400. 00000080:00000001:3.0:1775768213.170359:0:13471:0:(vvp_io.c:1812:vvp_io_init()) Process entered 02000000:00000001:0.0:1775768213.170363:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1775768213.170363:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000080:00200000:3.0:1775768213.170363:0:13471:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017b:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1775768213.170366:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768213.170368:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000100:00100000:2.0:1775768213.170371:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfc700 x1862022294288512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000001:3.0:1775768213.170371:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1775768213.170372:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 00000020:00000001:3.0:1775768213.170374:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 02000000:00000010:0.0:1775768213.170375:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1f800. 00000080:00000001:3.0:1775768213.170377:0:13471:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768213.170380:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768213.170383:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00020000:00000002:3.0:1775768213.170383:0:13471:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017b:0x0]io ffff9942077c02a8 type 6 ignore/verify layout 1/0 00000100:00000010:0.0:1775768213.170386:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfce00. 00000100:00000001:2.0:1775768213.170386:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775768213.170390:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:3.0:1775768213.170391:0:13471:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207c1f000) owner 0000000000000000/ffff994220080000 00000100:00000040:2.0:1775768213.170395:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a176dd1] to pc [ptlrpcd_rcv+-1] req@ffff994207dfc700 x1862022294288512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:3.0:1775768213.170396:0:13471:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00000100:00000001:0.0:1775768213.170397:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00020000:00000001:3.0:1775768213.170399:0:13471:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00000100:00000001:0.0:1775768213.170400:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00020000:00000001:3.0:1775768213.170402:0:13471:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00000100:00000001:0.0:1775768213.170404:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00020000:00000001:3.0:1775768213.170405:0:13471:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768213.170410:0:13471:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768213.170411:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768213.170411:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768213.170413:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768213.170415:0:13471:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00000100:00000001:0.0:1775768213.170417:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1775768213.170418:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5475) 00020000:00000001:3.0:1775768213.170418:0:13471:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768213.170419:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768213.170421:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1775768213.170423:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:3.0:1775768213.170426:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00000040:0.0:1775768213.170427:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dff480 x1862022294288128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:3.0:1775768213.170430:0:13471:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00020000:00000002:3.0:1775768213.170433:0:13471:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207c1f000) owner 0000000000000000/ffff994220080000 00000100:00000001:0.0:1775768213.170435:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768213.170437:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768213.170438:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768213.170439:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768213.170442:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1775768213.170442:0:13471:0:(cl_io.c:823:cl_io_loop()) Process entered 02000000:00000001:0.0:1775768213.170445:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768213.170445:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00020000:00000001:3.0:1775768213.170448:0:13471:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00000100:00100000:0.0:1775768213.170449:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294288128:192.168.202.156@tcp:38:kworker.0 00020000:00200000:3.0:1775768213.170451:0:13471:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00000001:0.0:1775768213.170454:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00020000:00000002:3.0:1775768213.170456:0:13471:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 02000000:00000001:0.0:1775768213.170457:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768213.170458:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768213.170461:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00020000:00000001:3.0:1775768213.170461:0:13471:0:(lov_io.c:155:lov_sub_get()) Process entered 00020000:00000001:3.0:1775768213.170464:0:13471:0:(lov_io.c:104:lov_io_sub_init()) Process entered 02000000:00000010:0.0:1775768213.170465:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220028400. 00000020:00000001:3.0:1775768213.170466:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 02000000:00000001:0.0:1775768213.170468:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1775768213.170471:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff99420693ecf0. 00000400:00000200:0.0:1775768213.170472:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000010:3.0:1775768213.170476:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dce400. 00000400:00000010:0.0:1775768213.170477:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775768213.170483:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000020:00000010:3.0:1775768213.170483:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210dce800. 00000100:00000200:0.0:1775768213.170488:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294288128, portal 10 00000020:00000010:3.0:1775768213.170489:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207981900. 00000100:00000001:0.0:1775768213.170490:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768213.170493:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611849856 : -112966097701760 : ffff994207dff480) 00000008:00000010:3.0:1775768213.170494:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994208b51650. 00020000:00000010:3.0:1775768213.170497:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994218000000. 00000100:00000040:0.0:1775768213.170498:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dff480 x1862022294288128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768238 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00008000:00000010:3.0:1775768213.170501:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994207e0ade8. 00000080:00000010:3.0:1775768213.170507:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99420541b0a8. 00000100:00000001:0.0:1775768213.170509:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000080:00000010:3.0:1775768213.170511:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077c1da8. 00000100:00000040:0.0:1775768213.170515:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000020:00000010:3.0:1775768213.170516:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcf600. 00000400:00000010:0.0:1775768213.170519:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000008:00000010:3.0:1775768213.170521:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942086f8160. 00000100:00000200:0.0:1775768213.170522:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294288128, offset 0 00020000:00000010:3.0:1775768213.170524:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994207e8ed00. 00008000:00000010:3.0:1775768213.170529:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994209270ef0. 00000400:00000200:0.0:1775768213.170530:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000080:00000010:3.0:1775768213.170533:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942090e3960. 00000400:00000200:0.0:1775768213.170537:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:3.0:1775768213.170538:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107590089976 : -112966119461640 : ffff99420693ecf8) 00000020:00001000:3.0:1775768213.170541:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff99420693ecf0 00000400:00000200:0.0:1775768213.170543:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000020:00000001:3.0:1775768213.170545:0:13471:0:(cl_object.c:151:cl_object_top()) ffff9942078a75f8 -> ffff9942078a7f58 00000400:00000200:0.0:1775768213.170548:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:3.0:1775768213.170549:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:3.0:1775768213.170552:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000400:00000200:0.0:1775768213.170553:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000020:00000001:3.0:1775768213.170554:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000400:00000200:0.0:1775768213.170557:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000020:00000001:3.0:1775768213.170558:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768213.170562:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00020000:00000001:3.0:1775768213.170562:0:13471:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768213.170567:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225424:225424:256:4294967295] 192.168.202.156@tcp LPNI seq info [225420:225420:8:4294967295] 00020000:00000001:3.0:1775768213.170567:0:13471:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107612435560 : -112966097116056 : ffff994207e8e468) 00000020:00000001:3.0:1775768213.170572:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000400:00000200:0.0:1775768213.170574:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:3.0:1775768213.170576:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768213.170580:0:13471:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768213.170582:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000020:00000001:3.0:1775768213.170583:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1775768213.170588:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000020:00000001:3.0:1775768213.170588:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00020000:00000001:3.0:1775768213.170590:0:13471:0:(lov_io.c:1108:lov_io_lock()) Process entered 00000800:00000010:0.0:1775768213.170592:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00020000:00000001:3.0:1775768213.170592:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775768213.170595:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00000800:00000200:0.0:1775768213.170598:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000001:3.0:1775768213.170598:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:3.0:1775768213.170601:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1775768213.170602:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000020:00000001:3.0:1775768213.170605:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775768213.170609:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00020000:00000001:3.0:1775768213.170610:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768213.170614:0:13471:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775768213.170614:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000020:00000001:3.0:1775768213.170615:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:3.0:1775768213.170618:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768213.170620:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1775768213.170623:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000020:00000001:3.0:1775768213.170624:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00020000:00000001:3.0:1775768213.170626:0:13471:0:(lov_io.c:1114:lov_io_start()) Process entered 00000800:00000010:0.0:1775768213.170627:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00020000:00000001:3.0:1775768213.170629:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775768213.170631:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00000400:00000200:0.0:1775768213.170632:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000008:00000001:3.0:1775768213.170633:0:13471:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000008:00000001:3.0:1775768213.170635:0:13471:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000008:00000001:3.0:1775768213.170638:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000400:00000200:0.0:1775768213.170639:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000008:00000020:3.0:1775768213.170642:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000400:00000200:0.0:1775768213.170644:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000008:00000001:3.0:1775768213.170645:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1775768213.170649:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000008:00000001:3.0:1775768213.170649:0:13471:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000008:00000001:3.0:1775768213.170651:0:13471:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000100:00000001:0.0:1775768213.170654:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000008:00000001:3.0:1775768213.170654:0:13471:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107884688256 : -112965824863360 : ffff994218232380) 00000008:00000020:3.0:1775768213.170658:0:13471:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994218232380 ready 0|-|- wr 1|-|+ rd 0|- 9 in flight 00000100:00000200:0.0:1775768213.170660:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dff480 x1862022294288128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768238 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1775768213.170663:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:3.0:1775768213.170666:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:3.0:1775768213.170668:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768213.170670:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000008:00000001:3.0:1775768213.170671:0:13471:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000100:00000040:0.0:1775768213.170674:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dff480 x1862022294288128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768238 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000020:3.0:1775768213.170675:0:13471:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994210674fc0 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000008:00000001:3.0:1775768213.170686:0:13471:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768213.170688:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1775768213.170689:0:13471:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768213.170691:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:3.0:1775768213.170693:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768213.170696:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768213.170696:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768213.170700:0:13471:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768213.170701:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000020:00000001:3.0:1775768213.170702:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768213.170707:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00000400:00000200:0.0:1775768213.170709:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:3.0:1775768213.170709:0:13471:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00020000:00000001:3.0:1775768213.170712:0:13471:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000100:00000001:0.0:1775768213.170713:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300138003 00000020:00000001:3.0:1775768213.170714:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000001:3.0:1775768213.170717:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00000100:00000001:0.0:1775768213.170718:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300139325 00020000:00000001:3.0:1775768213.170719:0:13471:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768213.170723:0:13471:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000100:00000001:0.0:1775768213.170724:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300138003 00000020:00000001:3.0:1775768213.170725:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00000100:00000001:0.0:1775768213.170727:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300139325 00000020:00000001:3.0:1775768213.170728:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00020000:00000001:3.0:1775768213.170730:0:13471:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00000100:00000001:0.0:1775768213.170731:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:3.0:1775768213.170732:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775768213.170734:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000100:00000001:0.0:1775768213.170735:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768213.170736:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000100:00000001:0.0:1775768213.170739:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00020000:00000001:3.0:1775768213.170740:0:13471:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768213.170743:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768213.170745:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768213/real 1775768213] req@ffff994207dff480 x1862022294288128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:3.0:1775768213.170746:0:13471:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000020:00000001:3.0:1775768213.170749:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:3.0:1775768213.170751:0:13471:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000020:00000001:3.0:1775768213.170754:0:13471:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000400:00000001:0.0:1775768213.170756:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:3.0:1775768213.170756:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:3.0:1775768213.170757:0:13471:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00000400:00000001:0.0:1775768213.170758:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00020000:00000001:3.0:1775768213.170759:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000400:00000010:0.0:1775768213.170760:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000020:00000001:3.0:1775768213.170761:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000001:3.0:1775768213.170763:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000400:00000200:0.0:1775768213.170765:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00020000:00000001:3.0:1775768213.170766:0:13471:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775768213.170767:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00020000:00000001:3.0:1775768213.170769:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768213.170770:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00020000:00000001:3.0:1775768213.170773:0:13471:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000100:00000200:0.0:1775768213.170775:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dff480 x1862022294288128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775768213.170775:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000020:00000001:3.0:1775768213.170778:0:13471:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768213.170783:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:3.0:1775768213.170785:0:13471:0:(lov_io.c:675:lov_io_fini()) Process entered 00000100:00000200:0.0:1775768213.170788:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dff480 x1862022294288128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:3.0:1775768213.170788:0:13471:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:3.0:1775768213.170790:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:3.0:1775768213.170793:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:3.0:1775768213.170795:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff99420693ecf0 00000020:00000010:3.0:1775768213.170800:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210dce800. 00000100:00000001:0.0:1775768213.170805:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000010:3.0:1775768213.170806:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207981900. 00000100:00000001:0.0:1775768213.170807:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768213.170811:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768213.170812:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00000010:3.0:1775768213.170813:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994208b51650. 00000100:00100000:0.0:1775768213.170814:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dff480 x1862022294288128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000010:3.0:1775768213.170818:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994218000000. 00008000:00000010:3.0:1775768213.170823:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994207e0ade8. 00000100:00000001:0.0:1775768213.170824:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000080:00000010:3.0:1775768213.170829:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99420541b0a8. 00000100:00000040:0.0:1775768213.170842:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dff480 x1862022294288128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768238 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000010:3.0:1775768213.170845:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077c1da8. 00000020:00000010:3.0:1775768213.170851:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dce400. 00000008:00000010:3.0:1775768213.170855:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942086f8160. 00000100:00000001:0.0:1775768213.170856:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00020000:00000010:3.0:1775768213.170860:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994207e8ed00. 00000100:00000001:0.0:1775768213.170861:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768213.170864:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00008000:00000010:3.0:1775768213.170864:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994209270ef0. 00000100:00000001:0.0:1775768213.170866:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768213.170869:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000080:00000010:3.0:1775768213.170869:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942090e3960. 00000100:00000001:0.0:1775768213.170874:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000010:3.0:1775768213.170874:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcf600. 00000020:00000010:3.0:1775768213.170878:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff99420693ecf0. 00000100:00080000:0.0:1775768213.170880:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00020000:00000001:3.0:1775768213.170882:0:13471:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00000100:00080000:0.0:1775768213.170884:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00020000:00000001:3.0:1775768213.170886:0:13471:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000100:00000001:0.0:1775768213.170887:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000080:00000001:3.0:1775768213.170889:0:13471:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000080:00200000:3.0:1775768213.170891:0:13471:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00000040:0.0:1775768213.170892:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dff480 x1862022294288128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768238 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000001:3.0:1775768213.170901:0:13471:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000100:00100000:0.0:1775768213.170904:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294288128:192.168.202.156@tcp:38:kworker.0 00000020:00000001:3.0:1775768213.170904:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:3.0:1775768213.170907:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff99420693e000 00000100:00000001:0.0:1775768213.170908:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000010:3.0:1775768213.170910:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210dcf000. 00000100:00000040:0.0:1775768213.170912:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfed80 x1862022294288256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1775768213.170915:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207981e40. 00000008:00000010:3.0:1775768213.170921:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994208b50000. 00000100:00000001:0.0:1775768213.170922:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768213.170924:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:3.0:1775768213.170926:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994218000300. 02000000:00000001:0.0:1775768213.170929:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00008000:00000010:3.0:1775768213.170930:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994207e08e20. 02000000:00000001:0.0:1775768213.170931:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768213.170935:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:3.0:1775768213.170935:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99420541b1f8. 00000100:00100000:0.0:1775768213.170941:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294288256:192.168.202.156@tcp:38:kworker.0 00000080:00000010:3.0:1775768213.170941:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077c0248. 00000020:00000010:3.0:1775768213.170946:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcf200. 00000100:00000001:0.0:1775768213.170947:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768213.170950:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000008:00000010:3.0:1775768213.170950:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942086f8420. 02000000:00000001:0.0:1775768213.170953:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:3.0:1775768213.170954:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994207e8e400. 02000000:00000001:0.0:1775768213.170956:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00008000:00000010:3.0:1775768213.170959:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994209270a90. 02000000:00000010:0.0:1775768213.170961:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002a800. 00000080:00000010:3.0:1775768213.170962:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942090e33c0. 02000000:00000001:0.0:1775768213.170965:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1775768213.170966:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcf800. 00000020:00000010:3.0:1775768213.170970:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff99420693e000. 00000400:00000200:0.0:1775768213.170971:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768213.170976:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775768213.170979:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768213.170981:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294288256, portal 10 00000100:00000001:0.0:1775768213.170983:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000080:00000001:3.0:1775768213.170983:0:13471:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768213.170985:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611848064 : -112966097703552 : ffff994207dfed80) 00000100:00000040:0.0:1775768213.170990:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfed80 x1862022294288256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768238 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000001:3.0:1775768213.170991:0:13471:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:3.0:1775768213.170996:0:13471:0:(rw.c:1481:ll_writepages()) Process entered 00000100:00000001:0.0:1775768213.170999:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000080:00000001:3.0:1775768213.170999:0:13471:0:(file.c:5250:cl_sync_file_range()) Process entered 00000020:00000001:3.0:1775768213.171001:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00000040:0.0:1775768213.171004:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000020:00000010:3.0:1775768213.171006:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff99420693e000. 00000400:00000010:0.0:1775768213.171008:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000020:00000010:3.0:1775768213.171010:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcf800. 00000100:00000200:0.0:1775768213.171011:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294288256, offset 0 00000020:00000010:3.0:1775768213.171014:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210dcf200. 00000400:00000200:0.0:1775768213.171018:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000020:00000010:3.0:1775768213.171019:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207981e40. 00000008:00000010:3.0:1775768213.171023:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994208b50000. 00000400:00000200:0.0:1775768213.171026:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000010:3.0:1775768213.171028:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994218000300. 00008000:00000010:3.0:1775768213.171032:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994207e08e20. 00000400:00000200:0.0:1775768213.171034:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000080:00000010:3.0:1775768213.171036:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99420541b1f8. 00000400:00000200:0.0:1775768213.171039:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000080:00000010:3.0:1775768213.171040:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077c0248. 00000400:00000200:0.0:1775768213.171044:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000020:00000010:3.0:1775768213.171044:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcf000. 00000400:00000200:0.0:1775768213.171047:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000008:00000010:3.0:1775768213.171048:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942086f8420. 00000400:00000200:0.0:1775768213.171050:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00020000:00000010:3.0:1775768213.171052:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994207e8e400. 00000400:00000200:0.0:1775768213.171055:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225425:225425:256:4294967295] 192.168.202.156@tcp LPNI seq info [225421:225421:8:4294967295] 00008000:00000010:3.0:1775768213.171056:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994209270a90. 00000400:00000200:0.0:1775768213.171061:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000080:00000010:3.0:1775768213.171061:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942090e33c0. 00000020:00000001:3.0:1775768213.171064:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107590086664 : -112966119464952 : ffff99420693e008) 00000400:00000200:0.0:1775768213.171068:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000020:00001000:3.0:1775768213.171069:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff99420693e000 00000020:00000001:3.0:1775768213.171075:0:13471:0:(cl_object.c:151:cl_object_top()) ffff99420791a850 -> ffff99420791a850 00000800:00000200:0.0:1775768213.171077:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000020:00000001:3.0:1775768213.171078:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:3.0:1775768213.171081:0:13471:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000800:00000010:0.0:1775768213.171084:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000080:00200000:3.0:1775768213.171084:0:13471:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017f:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:3.0:1775768213.171089:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000800:00000200:0.0:1775768213.171092:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000001:3.0:1775768213.171092:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:3.0:1775768213.171093:0:13471:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1775768213.171098:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00020000:00000002:3.0:1775768213.171098:0:13471:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017f:0x0]io ffff9942077c02a8 type 6 ignore/verify layout 1/0 00020000:00000002:3.0:1775768213.171104:0:13471:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207dfbd90) owner 0000000000000000/ffff994220080000 00000800:00000200:0.1:1775768213.171107:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00020000:00000001:3.0:1775768213.171108:0:13471:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00000800:00000200:0.1:1775768213.171110:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00020000:00000001:3.0:1775768213.171111:0:13471:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:3.0:1775768213.171114:0:13471:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00000800:00000100:0.0:1775768213.171115:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768213.171117:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00020000:00000001:3.0:1775768213.171117:0:13471:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768213.171120:0:13471:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1775768213.171121:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00020000:00000001:3.0:1775768213.171125:0:13471:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00000400:00000200:0.0:1775768213.171128:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00020000:00000001:3.0:1775768213.171128:0:13471:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768213.171131:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000020:00000001:3.0:1775768213.171131:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000400:00000010:0.0:1775768213.171134:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000020:00000001:3.0:1775768213.171134:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:3.0:1775768213.171136:0:13471:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00000100:00000001:0.0:1775768213.171137:0:2399:0:(events.c:34:request_out_callback()) Process entered 00020000:00000002:3.0:1775768213.171139:0:13471:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207dfbd90) owner 0000000000000000/ffff994220080000 00000100:00000200:0.0:1775768213.171142:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfed80 x1862022294288256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768238 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775768213.171142:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768213.171147:0:13471:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:3.0:1775768213.171149:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00020000:00000001:3.0:1775768213.171151:0:13471:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00020000:00200000:3.0:1775768213.171170:0:13471:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00000002:3.0:1775768213.171172:0:13471:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:3.0:1775768213.171175:0:13471:0:(lov_io.c:155:lov_sub_get()) Process entered 00000100:00000001:0.0:1775768213.171176:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00020000:00000001:3.0:1775768213.171176:0:13471:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000020:00000001:3.0:1775768213.171179:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00000040:0.0:1775768213.171182:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfed80 x1862022294288256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768238 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1775768213.171183:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff99420693ecf0. 00000020:00000010:3.0:1775768213.171187:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcf600. 00000020:00000010:3.0:1775768213.171192:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210dce400. 00000020:00000010:3.0:1775768213.171196:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207981900. 00000100:00000001:0.0:1775768213.171197:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768213.171200:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000008:00000010:3.0:1775768213.171200:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994208b51650. 00000100:00000001:0.0:1775768213.171204:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:3.0:1775768213.171204:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994218000000. 00008000:00000010:3.0:1775768213.171207:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994207e0ade8. 00000100:00000040:0.0:1775768213.171208:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:0.0:1775768213.171212:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000080:00000010:3.0:1775768213.171212:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99420541b0a8. 00000100:00000001:0.0:1775768213.171215:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300137996 00000080:00000010:3.0:1775768213.171216:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077c1da8. 00000100:00000001:0.0:1775768213.171218:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300139325 00000100:00000001:0.0:1775768213.171222:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300137996 00000020:00000010:3.0:1775768213.171222:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dce800. 00000100:00000001:0.0:1775768213.171226:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300139325 00000008:00000010:3.0:1775768213.171227:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942086f8160. 00000100:00000001:0.0:1775768213.171228:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768213.171232:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768213.171233:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00020000:00000010:3.0:1775768213.171233:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994207e8ed00. 00000100:00100000:0.0:1775768213.171237:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768213/real 1775768213] req@ffff994207dfed80 x1862022294288256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00008000:00000010:3.0:1775768213.171237:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994209270ef0. 00000080:00000010:3.0:1775768213.171242:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942090e3960. 00000020:00000001:3.0:1775768213.171246:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107590089976 : -112966119461640 : ffff99420693ecf8) 00000400:00000001:0.0:1775768213.171248:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768213.171250:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00001000:3.0:1775768213.171250:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff99420693ecf0 00000400:00000010:0.0:1775768213.171252:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000020:00000001:3.0:1775768213.171254:0:13471:0:(cl_object.c:151:cl_object_top()) ffff99420791a210 -> ffff99420791a850 00000400:00000200:0.0:1775768213.171256:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000020:00000001:3.0:1775768213.171256:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:3.0:1775768213.171258:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000400:00000010:0.0:1775768213.171259:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000020:00000001:3.0:1775768213.171259:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:3.0:1775768213.171261:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768213.171264:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00020000:00000001:3.0:1775768213.171264:0:13471:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768213.171267:0:13471:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107612435560 : -112966097116056 : ffff994207e8e468) 00000020:00000001:3.0:1775768213.171269:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000100:00000200:0.0:1775768213.171270:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfed80 x1862022294288256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775768213.171270:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768213.171273:0:13471:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768213.171276:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768213.171278:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00020000:00000001:3.0:1775768213.171280:0:13471:0:(lov_io.c:1108:lov_io_lock()) Process entered 00020000:00000001:3.0:1775768213.171282:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775768213.171284:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:3.0:1775768213.171285:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:3.0:1775768213.171287:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768213.171289:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768213.171291:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1775768213.171292:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfed80 x1862022294288256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:3.0:1775768213.171293:0:13471:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768213.171295:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:3.0:1775768213.171296:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768213.171299:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768213.171302:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00020000:00000001:3.0:1775768213.171304:0:13471:0:(lov_io.c:1114:lov_io_start()) Process entered 00020000:00000001:3.0:1775768213.171305:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775768213.171307:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00000100:00000001:0.0:1775768213.171310:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000008:00000001:3.0:1775768213.171310:0:13471:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000008:00000001:3.0:1775768213.171311:0:13471:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000100:00000001:0.0:1775768213.171313:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1775768213.171314:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000100:00000001:0.0:1775768213.171315:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000008:00000020:3.0:1775768213.171316:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000100:00000001:0.0:1775768213.171317:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1775768213.171318:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1775768213.171322:0:13471:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000100:00100000:0.0:1775768213.171325:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfed80 x1862022294288256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1775768213.171325:0:13471:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000008:00000001:3.0:1775768213.171326:0:13471:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107754938304 : -112965954613312 : ffff994210674fc0) 00000008:00000020:3.0:1775768213.171331:0:13471:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994210674fc0 ready 0|-|- wr 1|-|+ rd 0|- 9 in flight 00000008:00000001:3.0:1775768213.171337:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000100:00000001:0.0:1775768213.171339:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000008:00000020:3.0:1775768213.171339:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:3.0:1775768213.171342:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768213.171344:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfed80 x1862022294288256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768238 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1775768213.171345:0:13471:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000008:00000020:3.0:1775768213.171347:0:13471:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994218232380 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000008:00000001:3.0:1775768213.171354:0:13471:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1775768213.171357:0:13471:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768213.171359:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:3.0:1775768213.171359:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768213.171364:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00020000:00000001:3.0:1775768213.171364:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768213.171365:0:13471:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768213.171366:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:3.0:1775768213.171367:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768213.171368:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1775768213.171369:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00000100:00000001:0.0:1775768213.171370:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00020000:00000001:3.0:1775768213.171371:0:13471:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00020000:00000001:3.0:1775768213.171373:0:13471:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000020:00000001:3.0:1775768213.171374:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00000100:00000001:0.0:1775768213.171375:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:3.0:1775768213.171375:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00020000:00000001:3.0:1775768213.171377:0:13471:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768213.171378:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00020000:00000001:3.0:1775768213.171379:0:13471:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000020:00000001:3.0:1775768213.171381:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00000100:00080000:0.0:1775768213.171383:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000020:00000001:3.0:1775768213.171383:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00020000:00000001:3.0:1775768213.171385:0:13471:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00000100:00000001:0.0:1775768213.171386:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00020000:00000001:3.0:1775768213.171387:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775768213.171388:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000020:00000001:3.0:1775768213.171390:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:3.0:1775768213.171391:0:13471:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768213.171392:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfed80 x1862022294288256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768238 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:3.0:1775768213.171393:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768213.171396:0:13471:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000020:00000001:3.0:1775768213.171397:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:3.0:1775768213.171399:0:13471:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000020:00000001:3.0:1775768213.171401:0:13471:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000001:3.0:1775768213.171403:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:3.0:1775768213.171404:0:13471:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00000100:00100000:0.0:1775768213.171406:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294288256:192.168.202.156@tcp:38:kworker.0 00020000:00000001:3.0:1775768213.171406:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775768213.171409:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000001:3.0:1775768213.171410:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00020000:00000001:3.0:1775768213.171412:0:13471:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768213.171413:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00020000:00000001:3.0:1775768213.171414:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768213.171416:0:13471:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000100:00000040:0.0:1775768213.171417:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfe300 x1862022294288384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775768213.171418:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000020:00000001:3.0:1775768213.171420:0:13471:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768213.171423:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:3.0:1775768213.171425:0:13471:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:3.0:1775768213.171426:0:13471:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:3.0:1775768213.171428:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00000100:00000001:0.0:1775768213.171430:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000001:3.0:1775768213.171431:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000100:00000001:0.0:1775768213.171433:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:3.0:1775768213.171433:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff99420693ecf0 00000020:00000010:3.0:1775768213.171435:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210dce400. 02000000:00000001:0.0:1775768213.171436:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768213.171438:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1775768213.171439:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207981900. 02000000:00000001:0.0:1775768213.171441:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:3.0:1775768213.171442:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994208b51650. 00000100:00100000:0.0:1775768213.171445:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294288384:192.168.202.156@tcp:8:kworker.0 00020000:00000010:3.0:1775768213.171446:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994218000000. 00000100:00000001:0.0:1775768213.171449:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00008000:00000010:3.0:1775768213.171449:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994207e0ade8. 02000000:00000001:0.0:1775768213.171451:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000080:00000010:3.0:1775768213.171452:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99420541b0a8. 02000000:00000001:0.0:1775768213.171453:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768213.171456:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000080:00000010:3.0:1775768213.171456:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077c1da8. 00000020:00000010:3.0:1775768213.171459:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcf600. 02000000:00000010:0.0:1775768213.171460:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220029800. 00000008:00000010:3.0:1775768213.171461:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942086f8160. 02000000:00000001:0.0:1775768213.171462:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:3.0:1775768213.171465:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994207e8ed00. 00000400:00000200:0.0:1775768213.171468:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00008000:00000010:3.0:1775768213.171469:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994209270ef0. 00000400:00000010:0.0:1775768213.171471:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000080:00000010:3.0:1775768213.171472:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942090e3960. 00000400:00000010:0.0:1775768213.171474:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000020:00000010:3.0:1775768213.171475:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dce800. 00000100:00000200:0.0:1775768213.171477:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294288384, portal 4 00000020:00000010:3.0:1775768213.171478:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff99420693ecf0. 00000100:00000001:0.0:1775768213.171482:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00020000:00000001:3.0:1775768213.171482:0:13471:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00000100:00000001:0.0:1775768213.171484:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611845376 : -112966097706240 : ffff994207dfe300) 00020000:00000001:3.0:1775768213.171484:0:13471:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:3.0:1775768213.171486:0:13471:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000080:00200000:3.0:1775768213.171488:0:13471:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017f:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:3.0:1775768213.171490:0:13471:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:3.0:1775768213.171492:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000100:00000040:0.0:1775768213.171493:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfe300 x1862022294288384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768238 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00001000:3.0:1775768213.171493:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff99420693e000 00000020:00000010:3.0:1775768213.171496:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210dcf200. 00000020:00000010:3.0:1775768213.171499:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207981e40. 00000008:00000010:3.0:1775768213.171502:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994208b50000. 00020000:00000010:3.0:1775768213.171506:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994218000300. 00008000:00000010:3.0:1775768213.171509:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994207e08e20. 00000100:00000001:0.0:1775768213.171512:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000080:00000010:3.0:1775768213.171513:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99420541b1f8. 00000080:00000010:3.0:1775768213.171516:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077c0248. 00000100:00000040:0.0:1775768213.171520:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000020:00000010:3.0:1775768213.171520:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcf800. 00000008:00000010:3.0:1775768213.171522:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942086f8420. 00000400:00000010:0.0:1775768213.171525:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00020000:00000010:3.0:1775768213.171525:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994207e8e400. 00008000:00000010:3.0:1775768213.171530:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994209270a90. 00000100:00000200:0.0:1775768213.171531:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294288384, offset 0 00000080:00000010:3.0:1775768213.171532:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942090e33c0. 00000020:00000010:3.0:1775768213.171535:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcf000. 00000020:00000010:3.0:1775768213.171538:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff99420693e000. 00000400:00000200:0.0:1775768213.171541:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000080:00000001:3.0:1775768213.171542:0:13471:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:3.0:1775768213.171545:0:13471:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768213.171549:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768213.171554:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768213.171557:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768213.171561:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768213.171563:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768213.171566:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768213.171568:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225426:225426:256:4294967295] 192.168.202.156@tcp LPNI seq info [225422:225422:8:4294967295] 00000400:00000200:0.0:1775768213.171572:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768213.171577:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768213.171580:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768213.171582:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775768213.171585:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768213.171587:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775768213.171589:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768213.171591:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1775768213.171593:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768213.171594:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775768213.171596:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768213.171598:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768213.171600:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775768213.171601:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775768213.171602:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768213.171605:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfe300 x1862022294288384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768238 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768213.171609:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768213.171612:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfe300 x1862022294288384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768238 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768213.171618:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768213.171619:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768213.171622:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768213.171625:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:0.0:1775768213.171627:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768213.171629:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300138019 00000100:00000001:0.0:1775768213.171632:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300139325 00000100:00000001:0.0:1775768213.171634:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300138019 00000100:00000001:0.0:1775768213.171635:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300139325 00000100:00000001:0.0:1775768213.171637:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768213.171639:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768213.171641:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768213.171645:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768213/real 1775768213] req@ffff994207dfe300 x1862022294288384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768213.171653:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768213.171654:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768213.171655:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775768213.171657:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768213.171659:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768213.171661:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768213.171664:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfe300 x1862022294288384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768213.171669:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfe300 x1862022294288384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768213.171673:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768213.171674:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768213.171675:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768213.171675:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768213.171677:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfe300 x1862022294288384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768213.171681:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768213.171683:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfe300 x1862022294288384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768238 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768213.171690:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768213.171692:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768213.171694:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768213.171695:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768213.171696:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768213.171699:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768213.171702:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768213.171705:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768213.171707:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768213.171710:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfe300 x1862022294288384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768238 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768213.171718:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294288384:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768213.171721:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768213.171724:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfc700 x1862022294288512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768213.171731:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768213.171732:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768213.171734:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768213.171736:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768213.171737:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768213.171740:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294288512:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768213.171743:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768213.171744:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768213.171745:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768213.171746:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768213.171748:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220029400. 02000000:00000001:0.0:1775768213.171750:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768213.171752:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768213.171753:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775768213.171755:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768213.171757:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294288512, portal 4 00000100:00000001:0.0:1775768213.171758:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768213.171759:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611838208 : -112966097713408 : ffff994207dfc700) 00000100:00000040:0.0:1775768213.171762:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfc700 x1862022294288512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768238 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768213.171766:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768213.171767:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768213.171769:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775768213.171770:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294288512, offset 0 00000400:00000200:0.0:1775768213.171772:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768213.171774:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768213.171777:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768213.171779:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768213.171780:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768213.171781:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768213.171783:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768213.171785:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225427:225427:256:4294967295] 192.168.202.156@tcp LPNI seq info [225423:225423:8:4294967295] 00000400:00000200:0.0:1775768213.171787:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768213.171790:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768213.171793:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768213.171795:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775768213.171797:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768213.171799:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775768213.171801:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768213.171802:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1775768213.171804:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768213.171806:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775768213.171807:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768213.171809:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768213.171811:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775768213.171812:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775768213.171813:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768213.171815:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfc700 x1862022294288512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768238 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768213.171824:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768213.171827:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfc700 x1862022294288512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768238 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768213.171846:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768213.171848:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768213.171851:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768213.171853:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:0.0:1775768213.171856:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768213.171858:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300138022 00000100:00000001:0.0:1775768213.171860:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300139325 00000100:00000001:0.0:1775768213.171862:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300138022 00000100:00000001:0.0:1775768213.171864:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300139325 00000100:00000001:0.0:1775768213.171866:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768213.171868:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768213.171870:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768213.171874:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768213/real 1775768213] req@ffff994207dfc700 x1862022294288512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768213.171882:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768213.171883:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768213.171884:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775768213.171886:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768213.171887:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768213.171888:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768213.171890:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfc700 x1862022294288512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768213.171895:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfc700 x1862022294288512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768213.171899:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768213.171900:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768213.171901:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768213.171902:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768213.171904:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfc700 x1862022294288512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768213.171907:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768213.171909:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfc700 x1862022294288512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768238 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768213.171913:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768213.171914:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768213.171915:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768213.171915:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768213.171916:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768213.171919:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768213.171921:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768213.171924:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768213.171926:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768213.171929:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfc700 x1862022294288512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768238 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768213.171936:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294288512:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768213.171939:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768213.171941:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768213.171943:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dff480 x1862022294288128/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768238 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768213.171949:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768213.171951:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768213.171952:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220028400. 02000000:00000001:0.0:1775768213.171955:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768213.171956:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768213.171958:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:0.0:1775768213.171960:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768213.171961:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1fc00. 02000000:00000001:0.0:1775768213.171963:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768213.171965:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768213.171966:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dff480. 00000100:00000001:0.0:1775768213.171968:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768213.171970:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768213.171971:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768213.171974:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfed80 x1862022294288256/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768238 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768213.171980:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768213.171981:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768213.171982:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002a800. 02000000:00000001:0.0:1775768213.171983:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768213.171984:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768213.171985:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:0.0:1775768213.171986:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768213.171987:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1ec00. 02000000:00000001:0.0:1775768213.171989:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768213.171989:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768213.171990:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfed80. 00000100:00000001:0.0:1775768213.171992:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768213.171992:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768213.171993:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768213.171995:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfe300 x1862022294288384/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768238 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768213.172001:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768213.172002:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768213.172003:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220029800. 02000000:00000001:0.0:1775768213.172005:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768213.172006:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768213.172007:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:0.0:1775768213.172008:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768213.172009:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1f000. 02000000:00000001:0.0:1775768213.172011:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768213.172012:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768213.172013:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfe300. 00000100:00000001:0.0:1775768213.172015:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768213.172016:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768213.172018:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768213.172021:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfc700 x1862022294288512/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768238 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768213.172027:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768213.172028:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768213.172029:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220029400. 02000000:00000001:0.0:1775768213.172031:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768213.172032:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768213.172034:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:0.0:1775768213.172035:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768213.172036:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1f400. 02000000:00000001:0.0:1775768213.172038:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768213.172039:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768213.172040:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfc700. 00000100:00000001:0.0:1775768213.172042:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768213.172043:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768213.172044:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768213.172047:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768213.172048:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768213.172049:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775768213.172050:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768213.233250:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768213.233254:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768213.233258:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768213.233259:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768213.233262:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768213.233263:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768213.233265:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768213.233266:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768213.233269:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768213.233270:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768213.233319:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768213.233322:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768213.233325:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768213.297263:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768213.297269:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768213.297271:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768213.297276:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768213.297280:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768213.297283:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768213.297286:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768213.297288:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768213.297291:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768214.193243:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768214.193250:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768214.193253:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768214.193253:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768214.193254:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768214.193258:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:1.0:1775768214.193260:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775768214.193260:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768214.193261:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768214.193264:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768214.193266:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768214.193267:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1775768214.193269:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775768214.193270:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768214.193272:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768214.193273:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768214.193274:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768214.193277:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768214.193278:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768214.193281:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768214.193285:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768214.193285:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768214.193287:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768214.193290:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768214.193293:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768214.193296:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768214.193299:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768214.257235:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768214.257240:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768214.257245:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768214.257247:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768214.257250:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768214.257251:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768214.257254:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768214.257255:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768214.257258:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768214.257259:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768214.257311:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768214.257314:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768214.257317:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768214.321267:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768214.321273:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768214.321275:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768214.321280:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768214.321284:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768214.321288:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768214.321290:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768214.321292:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768214.321295:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768214.513226:0:15036:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4300139325, at 4300139328 00000100:00000040:1.0:1775768214.513226:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4300139325, at 4300139328 00000100:00000010:1.0:1775768214.513239:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3600. 00000100:00000010:0.0:1775768214.513241:0:15036:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3d80. 00000100:00000001:3.0:1775768215.217211:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768215.217217:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768215.217219:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768215.217220:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768215.217224:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:3.0:1775768215.217225:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775768215.217227:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768215.217229:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:0.0:1775768215.217231:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768215.217233:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768215.217235:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768215.217237:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768215.217238:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768215.217240:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768215.217244:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768215.217244:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768215.217247:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768215.217250:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768215.217363:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768215.217366:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768215.217368:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768215.217371:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775768215.217373:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768215.217376:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768215.217377:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768215.217379:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768215.217381:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768215.281262:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768215.281267:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768215.281273:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768215.281275:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768215.281278:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768215.281279:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768215.281282:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768215.281284:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768215.281286:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768215.281288:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768215.281347:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768215.281349:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768215.281352:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768215.345248:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768215.345253:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768215.345254:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768215.345258:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768215.345261:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768215.345263:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768215.345265:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768215.345267:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768215.345269:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768216.241234:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768216.241240:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768216.241242:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768216.241248:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775768216.241252:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768216.241257:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768216.241259:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768216.241261:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768216.241264:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768216.241291:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768216.241298:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768216.241299:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768216.241304:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768216.241307:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768216.241309:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768216.241311:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768216.241313:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768216.241323:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768216.241363:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768216.241365:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768216.241367:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768216.241369:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775768216.241373:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768216.241375:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768216.241376:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768216.241377:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768216.241380:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768216.305260:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768216.305267:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768216.305271:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768216.305272:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768216.305276:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768216.305276:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768216.305279:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768216.305280:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768216.305282:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768216.305283:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768216.305336:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768216.305339:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768216.305342:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768216.369284:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768216.369292:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768216.369295:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768216.369302:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768216.369307:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768216.369312:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768216.369315:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768216.369318:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768216.369321:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768217.265247:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768217.265256:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768217.265258:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768217.265265:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775768217.265270:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768217.265275:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768217.265279:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768217.265281:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768217.265285:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768217.265304:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768217.265312:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768217.265313:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768217.265314:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768217.265318:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:2.0:1775768217.265319:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:0.0:1775768217.265320:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768217.265325:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768217.265326:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768217.265331:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768217.265334:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768217.265334:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768217.265340:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768217.265342:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768217.265344:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768217.265346:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768217.265348:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768217.265352:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768217.329267:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768217.329273:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768217.329279:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768217.329282:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768217.329284:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768217.329287:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768217.329289:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768217.329290:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768217.329292:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768217.329293:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768217.329296:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768217.329297:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768217.329297:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768217.329298:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768217.329299:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768217.329300:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768217.329301:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768217.329302:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768217.329304:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768217.329305:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768217.329306:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768217.329307:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768217.329308:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768217.329308:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768217.329310:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768217.329311:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768217.329313:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768217.329313:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768217.329314:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768217.329315:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768217.329316:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768217.329316:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768217.329317:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768217.329318:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768217.329398:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768217.329404:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768217.329409:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768217.393254:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768217.393259:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768217.393260:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768217.393264:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768217.393267:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768217.393270:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768217.393271:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768217.393273:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768217.393275:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768218.289226:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768218.289226:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768218.289226:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768218.289233:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768218.289234:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768218.289234:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768218.289235:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768218.289237:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768218.289237:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768218.289242:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775768218.289242:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775768218.289242:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768218.289246:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768218.289247:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768218.289247:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768218.289249:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768218.289250:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768218.289250:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768218.289252:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768218.289254:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768218.289257:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768218.289259:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768218.289261:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768218.289264:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768218.289264:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768218.289266:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768218.289268:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768218.289273:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768218.289278:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775768218.289280:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775768218.289282:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768218.289286:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775768218.289291:0:13471:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5470 00000400:00000200:1.0:1775768218.289298:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768218.289303:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775768218.289306:0:13471:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775768218.289309:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768218.289311:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768218.289313:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768218.289315:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775768218.289317:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775768218.289318:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:1.0:1775768218.289321:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775768218.289322:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:1.0:1775768218.289324:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:1.0:1775768218.289325:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:1.0:1775768218.289327:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768218.289328:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768218.289329:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768218.289331:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775768218.289333:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:1.0:1775768218.289335:0:13471:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775768218.289337:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768218.289339:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775768218.289341:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768218.289342:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768218.289344:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775768218.289345:0:13471:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775768218.289350:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207889f80. 00000020:00000040:1.0:1775768218.289352:0:13471:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:1.0:1775768218.289355:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768218.289356:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768218.289359:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775768218.289361:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775768218.289364:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775768218.289370:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543c400. 00000100:00000001:1.0:1775768218.289375:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768218.289382:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207889f80 x1862022294288640/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775768218.289389:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775768218.289392:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768218.289395:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ed2274f4] to pc [ptlrpcd_rcv+-1] req@ffff994207889f80 x1862022294288640/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768218.289411:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768218.289413:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768218.289415:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775768218.289416:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775768218.289417:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768218.289419:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775768218.289420:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5470 00000400:00000200:1.0:1775768218.289423:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768218.289426:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775768218.289429:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775768218.289432:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768218.289434:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768218.289437:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768218.289438:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768218.289440:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:0.0:1775768218.289441:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768218.289444:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000001:1.0:1775768218.289445:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775768218.289447:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:1.0:1775768218.289450:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000040:0.0:1775768218.289451:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207889f80 x1862022294288640/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1775768218.289453:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:1.0:1775768218.289459:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000001:0.0:1775768218.289463:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000001:1.0:1775768218.289464:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000001:0.0:1775768218.289466:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768218.289470:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:1.0:1775768218.289471:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 02000000:00000001:0.0:1775768218.289473:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768218.289474:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1775768218.289476:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768218.289477:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00100000:0.0:1775768218.289481:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207889f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294288640:192.168.202.156@tcp:250:kworker.0 00000100:00000001:1.0:1775768218.289481:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775768218.289486:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00000001:0.0:1775768218.289487:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768218.289490:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:1.0:1775768218.289490:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775768218.289493:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768218.289495:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775768218.289498:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768218.289499:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768218.289500:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768218.289501:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775768218.289506:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1775768218.289508:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:1.0:1775768218.289509:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768218.289512:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768218.289515:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220029400. 00000100:00000010:1.0:1775768218.289516:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff99420788b100. 02000000:00000001:0.0:1775768218.289517:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1775768218.289521:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000400:00000200:0.0:1775768218.289525:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:1.0:1775768218.289526:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000010:0.0:1775768218.289529:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 02000000:00000001:1.0:1775768218.289529:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768218.289533:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000010:0.0:1775768218.289536:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 02000000:00000001:1.0:1775768218.289536:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775768218.289538:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1775768218.289543:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294288640, portal 25 02000000:00000010:1.0:1775768218.289545:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543d800. 00000100:00000001:0.0:1775768218.289546:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768218.289549:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606126464 : -112966103425152 : ffff994207889f80) 00000100:00000001:1.0:1775768218.289551:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768218.289554:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207889f80 x1862022294288640/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768243 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768218.289560:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff99420788b100 x1862022294288768/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768218.289561:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768218.289567:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768218.289571:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000001:1.0:1775768218.289573:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000200:0.0:1775768218.289574:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294288640, offset 0 00000100:00000001:1.0:1775768218.289576:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768218.289580:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e26cf977] to pc [ptlrpcd_rcv+-1] req@ffff99420788b100 x1862022294288768/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775768218.289583:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00000001:1.0:1775768218.289590:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768218.289593:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00080000:1.0:1775768218.289595:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768218.289597:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000200:0.0:1775768218.289599:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00000001:1.0:1775768218.289600:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775768218.289603:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768218.289605:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:0.0:1775768218.289606:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:1.0:1775768218.289609:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5470 00000400:00000200:0.0:1775768218.289612:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768218.289614:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768218.289616:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768218.289619:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225428:225428:256:4294967295] 192.168.202.156@tcp LPNI seq info [225424:225424:8:4294967295] 00000400:00000200:0.0:1775768218.289621:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768218.289628:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768218.289632:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768218.289636:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000800:00000010:0.0:1775768218.289637:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000400:00000200:1.0:1775768218.289642:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000800:00000200:0.0:1775768218.289646:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00080000:1.0:1775768218.289648:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000800:00000200:0.0:1775768218.289650:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000100:00000040:1.0:1775768218.289653:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768218.289656:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768218.289658:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000800:00000200:0.1:1775768218.289662:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:1.0:1775768218.289663:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000800:00000200:0.1:1775768218.289667:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000020:00000001:1.0:1775768218.289668:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775768218.289670:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:1.0:1775768218.289672:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000800:00000100:0.0:1775768218.289674:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000020:00000040:1.0:1775768218.289675:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000800:00000010:0.0:1775768218.289677:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000020:00000001:1.0:1775768218.289677:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:1.0:1775768218.289680:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000400:00000200:0.0:1775768218.289681:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:1.0:1775768218.289684:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768218.289686:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1775768218.289689:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000100:00000040:1.0:1775768218.289691:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768218.289693:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000200:0.0:1775768218.289694:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775768218.289696:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000020:00000040:1.0:1775768218.289696:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:1.0:1775768218.289700:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768218.289702:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1775768218.289702:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768218.289705:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000200:0.0:1775768218.289706:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207889f80 x1862022294288640/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768243 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1775768218.289708:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768218.289713:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768218.289716:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775768218.289719:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1775768218.289720:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000008:00000001:1.0:1775768218.289723:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768218.289724:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207889f80 x1862022294288640/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768243 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768218.289727:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775768218.289733:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207888700. 00000100:00000001:0.0:1775768218.289734:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1775768218.289736:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000100:00000001:0.0:1775768218.289737:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768218.289743:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768218.289746:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000200:0.0:1775768218.289748:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:1.0:1775768218.289750:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768218.289751:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4300144445 00000100:00000001:1.0:1775768218.289753:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775768218.289756:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775768218.289757:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768218.289759:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942078a3d80. 02000000:00000010:1.0:1775768218.289761:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543f800. 00000100:00000001:1.0:1775768218.289764:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768218.289768:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00100000:1.0:1775768218.289768:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207888700 x1862022294288896/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775768218.289775:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775768218.289776:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768218.289778:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009f2ba301] to pc [ptlrpcd_rcv+-1] req@ffff994207888700 x1862022294288896/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768218.289791:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4300144445 00000100:00000001:1.0:1775768218.289795:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768218.289798:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942078a3120. 00000100:00080000:1.0:1775768218.289798:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775768218.289803:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00080000:1.0:1775768218.289803:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775768218.289805:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1775768218.289807:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:1.0:1775768218.289807:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768218.289812:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768218.289812:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:0.0:1775768218.289816:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00080000:1.0:1775768218.289816:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5470 00000400:00000200:1.0:1775768218.289822:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768218.289825:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00100000:0.0:1775768218.289826:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768218/real 1775768218] req@ffff994207889f80 x1862022294288640/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768218.289826:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775768218.289831:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768218.289835:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768218.289838:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768218.289843:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775768218.289846:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775768218.289847:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:1.0:1775768218.289849:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775768218.289851:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:1.0:1775768218.289854:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:1.0:1775768218.289858:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:1.0:1775768218.289863:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768218.289866:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000001:0.0:1775768218.289868:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:1.0:1775768218.289869:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000001:0.0:1775768218.289872:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1775768218.289872:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775768218.289875:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000400:00000010:0.0:1775768218.289876:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000100:00080000:1.0:1775768218.289877:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000400:00000200:0.0:1775768218.289881:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000100:00000001:1.0:1775768218.289881:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768218.289882:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:0.0:1775768218.289885:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 02000000:00000001:1.0:1775768218.289886:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768218.289888:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768218.289889:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775768218.289891:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:1.0:1775768218.289892:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768218.289894:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775768218.289898:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207889880. 00000020:00000040:1.0:1775768218.289900:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:1.0:1775768218.289902:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1775768218.289903:0:2399:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:1.0:1775768218.289903:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768218.289906:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775768218.289907:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775768218.289910:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1775768218.289911:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207889f80 x1862022294288640/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1775768218.289916:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543fc00. 00000100:00000001:1.0:1775768218.289922:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1775768218.289925:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207889f80 x1862022294288640/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768218.289930:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207889880 x1862022294289024/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768218.289940:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768218.289943:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768218.289947:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768218.289949:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768218.289950:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768218.289952:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768218.289957:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207889f80 x1862022294288640/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775768218.289960:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f11856b8] to pc [ptlrpcd_rcv+-1] req@ffff994207889880 x1862022294289024/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768218.289978:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768218.289979:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768218.289982:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775768218.289987:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207889f80 x1862022294288640/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768243 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768218.289987:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775768218.289988:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775768218.289990:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768218.289995:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:0.0:1775768218.290001:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768218.290001:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5470 00000100:00000001:0.0:1775768218.290005:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768218.290009:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000400:00000200:1.0:1775768218.290009:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768218.290011:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768218.290014:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768218.290017:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000400:00000200:1.0:1775768218.290017:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768218.290022:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768218.290022:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775768218.290028:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00080000:0.0:1775768218.290030:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768218.290033:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1775768218.290034:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768218.290039:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000040:0.0:1775768218.290042:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207889f80 x1862022294288640/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768243 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768218.290044:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775768218.290047:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775768218.290049:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:1.0:1775768218.290052:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775768218.290055:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:1.0:1775768218.290059:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:1.0:1775768218.290061:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00100000:0.0:1775768218.290063:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207889f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294288640:192.168.202.156@tcp:250:kworker.0 00000100:00000040:1.0:1775768218.290066:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768218.290070:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768218.290072:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768218.290073:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768218.290074:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:0.0:1775768218.290077:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000020:00000040:1.0:1775768218.290077:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:1.0:1775768218.290083:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000040:0.0:1775768218.290084:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207889f80 x1862022294288640/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768243 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768218.290085:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768218.290088:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775768218.290090:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768218.290095:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768218.290097:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 00000100:00000001:1.0:1775768218.290097:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775768218.290100:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 02000000:00000001:0.0:1775768218.290101:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000008:00000001:1.0:1775768218.290103:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768218.290104:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220029400. 00000100:00000001:1.0:1775768218.290105:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768218.290108:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000010:1.0:1775768218.290109:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207888e00. 00000020:00000001:0.0:1775768218.290110:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768218.290112:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000040:1.0:1775768218.290113:0:13471:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:0.0:1775768218.290115:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000001:1.0:1775768218.290116:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000010:0.0:1775768218.290118:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543c400. 02000000:00000001:1.0:1775768218.290118:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768218.290123:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768218.290124:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768218.290124:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768218.290127:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:1.0:1775768218.290127:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768218.290129:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207889f80. 02000000:00000010:1.0:1775768218.290132:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543e800. 00000100:00000001:0.0:1775768218.290133:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768218.290135:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768218.290137:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768218.290137:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768218.290144:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207888e00 x1862022294289152/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768218.290145:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768218.290147:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768218.290151:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775768218.290153:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768218.290178:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768218.290183:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff99420788b100 x1862022294288768/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768218.290184:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775768218.290188:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768218.290195:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000040:1.0:1775768218.290196:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001812f966] to pc [ptlrpcd_rcv+-1] req@ffff994207888e00 x1862022294289152/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768218.290197:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768218.290202:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768218.290204:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768218.290208:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768218.290211:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768218.290212:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff99420788b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294288768:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768218.290217:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768218.290218:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768218.290219:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768218.290219:0:13471:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5480) 02000000:00000001:0.0:1775768218.290224:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768218.290226:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220029400. 02000000:00000001:0.0:1775768218.290227:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768218.290230:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768218.290232:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775768218.290235:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768218.290237:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294288768, portal 10 00000100:00000001:0.0:1775768218.290238:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768218.290239:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606130944 : -112966103420672 : ffff99420788b100) 00000100:00000040:0.0:1775768218.290244:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff99420788b100 x1862022294288768/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768243 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768218.290248:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768218.290249:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768218.290250:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775768218.290252:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294288768, offset 0 00000400:00000200:0.0:1775768218.290254:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768218.290257:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768218.290259:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768218.290261:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768218.290262:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768218.290264:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768218.290266:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768218.290267:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225429:225429:256:4294967295] 192.168.202.156@tcp LPNI seq info [225425:225425:8:4294967295] 00000400:00000200:0.0:1775768218.290270:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768218.290275:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768218.290277:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768218.290279:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775768218.290282:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768218.290284:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775768218.290286:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768218.290288:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1775768218.290290:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768218.290291:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775768218.290294:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768218.290297:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768218.290300:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775768218.290301:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775768218.290303:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768218.290306:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff99420788b100 x1862022294288768/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768243 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768218.290313:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768218.290316:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff99420788b100 x1862022294288768/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768243 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768218.290321:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768218.290323:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768218.290326:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768218.290329:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:0.0:1775768218.290332:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768218.290334:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300143123 00000100:00000001:0.0:1775768218.290336:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300144445 00000100:00000001:0.0:1775768218.290338:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300143123 00000100:00000001:0.0:1775768218.290340:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300144445 00000100:00000001:0.0:1775768218.290341:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768218.290343:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768218.290344:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768218.290346:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768218/real 1775768218] req@ffff99420788b100 x1862022294288768/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768218.290351:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768218.290352:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768218.290354:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775768218.290355:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768218.290356:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768218.290358:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768218.290360:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff99420788b100 x1862022294288768/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768218.290364:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff99420788b100 x1862022294288768/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768218.290368:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768218.290368:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768218.290369:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768218.290370:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768218.290372:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff99420788b100 x1862022294288768/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768218.290375:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768218.290377:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff99420788b100 x1862022294288768/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768243 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768218.290381:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768218.290382:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768218.290383:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768218.290384:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768218.290385:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768218.290386:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768218.290387:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768218.290389:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768218.290391:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768218.290393:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff99420788b100 x1862022294288768/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768243 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768218.290397:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff99420788b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294288768:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768218.290400:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768218.290402:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207888700 x1862022294288896/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768218.290406:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768218.290406:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768218.290408:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768218.290409:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768218.290410:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768218.290411:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207888700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294288896:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768218.290413:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768218.290414:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768218.290415:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768218.290416:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768218.290418:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220029800. 02000000:00000001:0.0:1775768218.290420:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768218.290421:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768218.290423:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775768218.290425:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768218.290426:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294288896, portal 10 00000100:00000001:0.0:1775768218.290427:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768218.290428:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606120192 : -112966103431424 : ffff994207888700) 00000100:00000040:0.0:1775768218.290432:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207888700 x1862022294288896/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768243 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768218.290436:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768218.290437:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768218.290438:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775768218.290439:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294288896, offset 0 00000400:00000200:0.0:1775768218.290441:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768218.290444:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768218.290446:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768218.290447:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768218.290449:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768218.290451:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768218.290452:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768218.290454:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225430:225430:256:4294967295] 192.168.202.156@tcp LPNI seq info [225426:225426:8:4294967295] 00000400:00000200:0.0:1775768218.290456:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768218.290460:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768218.290462:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768218.290464:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775768218.290466:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768218.290468:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775768218.290470:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768218.290472:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1775768218.290474:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768218.290475:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775768218.290477:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768218.290479:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768218.290481:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775768218.290482:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775768218.290483:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768218.290485:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207888700 x1862022294288896/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768243 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768218.290489:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768218.290491:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207888700 x1862022294288896/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768243 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768218.290495:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768218.290495:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768218.290497:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768218.290498:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:0.0:1775768218.290500:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768218.290501:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300143115 00000100:00000001:0.0:1775768218.290502:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300144445 00000100:00000001:0.0:1775768218.290503:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300143115 00000100:00000001:0.0:1775768218.290505:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300144445 00000100:00000001:0.0:1775768218.290506:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768218.290507:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768218.290508:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768218.290510:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768218/real 1775768218] req@ffff994207888700 x1862022294288896/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768218.290514:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768218.290515:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768218.290516:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775768218.290517:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768218.290517:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768218.290518:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768218.290520:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207888700 x1862022294288896/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768218.290524:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207888700 x1862022294288896/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768218.290528:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768218.290529:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768218.290530:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768218.290530:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768218.290532:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207888700 x1862022294288896/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768218.290535:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768218.290537:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207888700 x1862022294288896/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768243 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768218.290541:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768218.290542:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768218.290543:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768218.290543:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768218.290545:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768218.290546:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768218.290548:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768218.290550:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768218.290551:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768218.290553:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207888700 x1862022294288896/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768243 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768218.290557:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207888700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294288896:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768218.290559:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768218.290561:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207889880 x1862022294289024/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768218.290565:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768218.290565:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768218.290566:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768218.290567:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768218.290568:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768218.290570:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207889880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294289024:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768218.290571:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768218.290572:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768218.290573:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768218.290574:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768218.290575:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002a800. 02000000:00000001:0.0:1775768218.290576:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768218.290578:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768218.290579:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775768218.290581:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768218.290582:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294289024, portal 4 00000100:00000001:0.0:1775768218.290583:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768218.290584:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606124672 : -112966103426944 : ffff994207889880) 00000100:00000040:0.0:1775768218.290587:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207889880 x1862022294289024/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768243 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768218.290591:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768218.290592:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768218.290593:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775768218.290595:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294289024, offset 0 00000400:00000200:0.0:1775768218.290596:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768218.290598:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768218.290600:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768218.290601:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768218.290602:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768218.290604:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768218.290605:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768218.290607:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225431:225431:256:4294967295] 192.168.202.156@tcp LPNI seq info [225427:225427:8:4294967295] 00000400:00000200:0.0:1775768218.290609:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768218.290613:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768218.290615:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768218.290616:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775768218.290618:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768218.290619:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775768218.290622:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768218.290623:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1775768218.290626:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768218.290627:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775768218.290628:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768218.290630:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768218.290632:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775768218.290633:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775768218.290634:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768218.290636:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207889880 x1862022294289024/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768243 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768218.290640:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768218.290642:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207889880 x1862022294289024/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768243 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768218.290649:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768218.290650:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768218.290652:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768218.290653:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:0.0:1775768218.290655:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768218.290656:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300143138 00000100:00000001:0.0:1775768218.290657:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300144445 00000100:00000001:0.0:1775768218.290658:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300143138 00000100:00000001:0.0:1775768218.290659:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300144445 00000100:00000001:0.0:1775768218.290660:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768218.290662:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768218.290662:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768218.290664:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768218/real 1775768218] req@ffff994207889880 x1862022294289024/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768218.290668:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768218.290669:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768218.290669:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775768218.290670:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768218.290671:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768218.290672:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768218.290674:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207889880 x1862022294289024/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768218.290678:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207889880 x1862022294289024/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768218.290681:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768218.290682:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768218.290683:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768218.290684:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768218.290686:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207889880 x1862022294289024/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768218.290690:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768218.290692:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207889880 x1862022294289024/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768243 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768218.290695:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768218.290697:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768218.290697:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768218.290698:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768218.290699:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768218.290701:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768218.290702:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768218.290704:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768218.290705:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768218.290706:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207889880 x1862022294289024/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768243 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768218.290710:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207889880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294289024:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768218.290712:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768218.290713:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768218.290715:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff99420788b100 x1862022294288768/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768243 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768218.290719:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768218.290720:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768218.290721:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220029400. 02000000:00000001:0.0:1775768218.290722:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768218.290723:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768218.290723:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:0.0:1775768218.290724:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768218.290725:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543d800. 02000000:00000001:0.0:1775768218.290728:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768218.290728:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768218.290729:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff99420788b100. 00000100:00000001:0.0:1775768218.290731:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768218.290732:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768218.290733:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768218.290734:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207888700 x1862022294288896/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768243 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768218.290740:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768218.290741:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768218.290742:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220029800. 02000000:00000001:0.0:1775768218.290744:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768218.290745:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768218.290746:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:0.0:1775768218.290748:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768218.290749:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543f800. 02000000:00000001:0.0:1775768218.290751:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768218.290752:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768218.290753:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207888700. 00000100:00000001:0.0:1775768218.290755:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768218.290756:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768218.290757:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768218.290760:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207889880 x1862022294289024/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768243 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768218.290766:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768218.290767:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768218.290768:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002a800. 02000000:00000001:0.0:1775768218.290771:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768218.290772:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768218.290773:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:0.0:1775768218.290775:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768218.290776:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543fc00. 02000000:00000001:0.0:1775768218.290778:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768218.290779:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768218.290780:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207889880. 00000100:00000001:0.0:1775768218.290794:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768218.290796:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768218.290797:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768218.290801:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768218.290801:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768218.290803:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775768218.290804:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768218.290805:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768218.290807:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207888e00 x1862022294289152/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768218.290811:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768218.290812:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768218.290814:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768218.290816:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768218.290817:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768218.290820:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207888e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294289152:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768218.290823:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768218.290825:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768218.290826:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768218.290828:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768218.290831:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002a800. 02000000:00000001:0.0:1775768218.290833:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768218.290836:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768218.290838:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775768218.290841:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768218.290844:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294289152, portal 4 00000100:00000001:0.0:1775768218.290845:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768218.290846:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606121984 : -112966103429632 : ffff994207888e00) 00000100:00000040:0.0:1775768218.290851:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207888e00 x1862022294289152/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768243 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768218.290857:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768218.290858:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768218.290860:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775768218.290862:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294289152, offset 0 00000400:00000200:0.0:1775768218.290866:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768218.290869:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768218.290872:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768218.290875:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768218.290877:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768218.290879:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768218.290881:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768218.290883:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225432:225432:256:4294967295] 192.168.202.156@tcp LPNI seq info [225428:225428:8:4294967295] 00000400:00000200:0.0:1775768218.290887:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768218.290893:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768218.290897:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768218.290899:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775768218.290903:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768218.290904:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775768218.290907:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768218.290908:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1775768218.290911:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768218.290912:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775768218.290914:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768218.290916:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768218.290917:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775768218.290919:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775768218.290920:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768218.290922:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207888e00 x1862022294289152/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768243 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768218.290927:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768218.290928:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207888e00 x1862022294289152/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768243 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768218.290932:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768218.290933:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768218.290935:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768218.290937:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:0.0:1775768218.290939:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768218.290940:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300143141 00000100:00000001:0.0:1775768218.290942:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300144445 00000100:00000001:0.0:1775768218.290943:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300143141 00000100:00000001:0.0:1775768218.290944:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300144445 00000100:00000001:0.0:1775768218.290946:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768218.290947:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768218.290949:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768218.290952:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768218/real 1775768218] req@ffff994207888e00 x1862022294289152/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768218.290958:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768218.290959:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768218.290961:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775768218.290963:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768218.290964:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768218.290966:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768218.290968:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207888e00 x1862022294289152/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768218.290976:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207888e00 x1862022294289152/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768218.290982:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768218.290983:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768218.290985:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768218.290986:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768218.290989:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207888e00 x1862022294289152/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768218.290996:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768218.290999:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207888e00 x1862022294289152/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768243 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768218.291005:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768218.291007:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768218.291008:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768218.291009:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768218.291010:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768218.291012:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768218.291014:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768218.291016:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768218.291017:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768218.291020:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207888e00 x1862022294289152/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768243 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768218.291024:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207888e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294289152:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768218.291026:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768218.291027:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768218.291028:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207888e00 x1862022294289152/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768243 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768218.291032:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768218.291033:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768218.291034:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002a800. 02000000:00000001:0.0:1775768218.291035:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768218.291036:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768218.291037:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:0.0:1775768218.291038:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768218.291039:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543e800. 02000000:00000001:0.0:1775768218.291040:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768218.291041:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768218.291042:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207888e00. 00000100:00000001:0.0:1775768218.291044:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768218.291044:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768218.291045:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768218.291048:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768218.291049:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768218.291050:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775768218.291051:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768218.353225:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768218.353229:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768218.353233:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768218.353233:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768218.353236:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768218.353237:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768218.353239:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768218.353239:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768218.353242:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768218.353242:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768218.353289:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768218.353291:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768218.353292:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768218.417261:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768218.417266:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768218.417268:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768218.417272:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768218.417275:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768218.417278:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768218.417280:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768218.417281:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768218.417283:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768219.313240:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768219.313245:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768219.313246:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768219.313250:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775768219.313253:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768219.313257:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768219.313259:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768219.313261:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768219.313264:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768219.313282:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768219.313289:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768219.313292:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768219.313298:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775768219.313302:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768219.313314:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768219.313318:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768219.313321:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768219.313328:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768219.313357:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768219.313361:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768219.313363:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768219.313365:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775768219.313368:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768219.313370:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768219.313372:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768219.313374:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768219.313377:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768219.377281:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768219.377289:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768219.377296:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768219.377297:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768219.377302:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768219.377303:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768219.377307:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768219.377308:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768219.377312:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768219.377314:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768219.377367:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768219.377371:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768219.377374:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768219.441277:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768219.441283:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768219.441285:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768219.441291:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768219.441295:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768219.441300:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768219.441303:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768219.441306:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768219.441309:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768219.633293:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4300144445, at 4300144448 00000100:00000040:0.0:1775768219.633304:0:15036:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4300144445, at 4300144448 00000100:00000010:1.0:1775768219.633307:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3120. 00000100:00000010:0.0:1775768219.633321:0:15036:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3d80. 00000100:00000001:3.0:1775768220.337201:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768220.337209:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768220.337211:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768220.337217:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768220.337228:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768220.337228:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768220.337228:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768220.337233:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768220.337233:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768220.337235:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768220.337235:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768220.337235:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768220.337239:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775768220.337239:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768220.337240:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768220.337243:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768220.337243:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768220.337244:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768220.337247:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768220.337247:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768220.337248:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768220.337251:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768220.337253:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768220.337256:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768220.337256:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768220.337258:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768220.337260:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768220.401246:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768220.401252:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768220.401257:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768220.401258:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768220.401262:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768220.401263:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768220.401266:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768220.401267:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768220.401270:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768220.401272:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768220.401332:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768220.401335:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768220.401338:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768220.465268:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768220.465273:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768220.465274:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768220.465278:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768220.465280:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768220.465283:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768220.465285:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768220.465287:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768220.465289:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768221.361258:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768221.361258:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768221.361265:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768221.361265:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768221.361270:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768221.361270:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768221.361277:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768221.361277:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:2.0:1775768221.361277:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768221.361282:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768221.361284:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768221.361284:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768221.361285:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768221.361290:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768221.361291:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768221.361291:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768221.361292:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768221.361297:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768221.361297:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768221.361300:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768221.361303:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768221.361304:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768221.361307:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768221.361310:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768221.361311:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768221.361314:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768221.361317:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768221.425272:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768221.425278:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768221.425284:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768221.425286:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768221.425291:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768221.425292:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768221.425295:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768221.425297:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768221.425300:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768221.425301:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768221.425346:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768221.425348:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768221.425350:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768221.489222:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768221.489228:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768221.489230:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768221.489235:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768221.489239:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768221.489242:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768221.489245:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768221.489247:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768221.489250:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768222.385246:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768222.385246:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768222.385247:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768222.385251:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768222.385252:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768222.385253:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768222.385253:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768222.385256:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768222.385259:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:0.0:1775768222.385260:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768222.385262:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768222.385263:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768222.385265:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768222.385266:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768222.385267:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:0.0:1775768222.385268:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768222.385269:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768222.385272:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768222.385273:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768222.385274:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768222.385275:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768222.385277:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768222.385278:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768222.385282:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768222.385282:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768222.385286:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768222.385289:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768222.449270:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768222.449277:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768222.449279:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768222.449283:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768222.449288:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768222.449290:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768222.449292:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768222.449295:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768222.449298:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768222.449300:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768222.449303:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768222.449304:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768222.449307:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768222.449308:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768222.449316:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768222.449317:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768222.449375:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768222.449378:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768222.449381:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768222.513271:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768222.513277:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768222.513279:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768222.513284:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768222.513288:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768222.513291:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768222.513294:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768222.513296:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768222.513299:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768223.409232:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768223.409232:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768223.409238:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768223.409238:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768223.409241:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768223.409241:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768223.409246:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775768223.409246:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00080000:1.0:1775768223.409247:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775768223.409251:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768223.409251:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:1.0:1775768223.409253:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775768223.409256:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1775768223.409256:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768223.409257:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768223.409257:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768223.409260:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:1.0:1775768223.409261:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775768223.409262:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768223.409263:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768223.409266:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768223.409267:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00100000:3.0:1775768223.409268:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768223.409269:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768223.409269:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768223.409272:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768223.409273:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:1.0:1775768223.409275:0:13471:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5475 00000100:00000001:3.0:1775768223.409276:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768223.409277:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768223.409281:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000400:00000200:1.0:1775768223.409284:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775768223.409284:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768223.409288:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768223.409292:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775768223.409295:0:13471:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775768223.409298:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768223.409301:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768223.409304:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768223.409307:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775768223.409311:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775768223.409314:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:1.0:1775768223.409317:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775768223.409320:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:1.0:1775768223.409323:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:1.0:1775768223.409326:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:1.0:1775768223.409330:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768223.409332:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768223.409336:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768223.409339:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775768223.409342:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:1.0:1775768223.409346:0:13471:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775768223.409350:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768223.409353:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775768223.409356:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768223.409359:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768223.409361:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775768223.409363:0:13471:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775768223.409372:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207888e00. 00000020:00000040:1.0:1775768223.409376:0:13471:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:1.0:1775768223.409379:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768223.409381:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768223.409383:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775768223.409385:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775768223.409389:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775768223.409397:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543dc00. 00000100:00000001:1.0:1775768223.409403:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768223.409410:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207888e00 x1862022294289280/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775768223.409423:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775768223.409427:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768223.409430:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001812f966] to pc [ptlrpcd_rcv+-1] req@ffff994207888e00 x1862022294289280/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768223.409443:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768223.409445:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768223.409447:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775768223.409448:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775768223.409449:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768223.409451:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775768223.409452:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5475 00000400:00000200:1.0:1775768223.409455:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768223.409457:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775768223.409458:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775768223.409460:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768223.409462:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768223.409464:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768223.409465:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775768223.409467:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775768223.409468:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:1.0:1775768223.409469:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775768223.409471:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:1.0:1775768223.409472:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:1.0:1775768223.409474:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000001:0.0:1775768223.409475:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1775768223.409476:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768223.409480:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1775768223.409481:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1775768223.409481:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768223.409484:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:0.0:1775768223.409485:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000040:1.0:1775768223.409486:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00080000:1.0:1775768223.409490:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775768223.409491:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768223.409493:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:0.0:1775768223.409495:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207888e00 x1862022294289280/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1775768223.409497:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768223.409501:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768223.409505:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775768223.409509:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1775768223.409510:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768223.409513:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1775768223.409514:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768223.409517:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768223.409518:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000010:1.0:1775768223.409520:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207889880. 02000000:00000001:0.0:1775768223.409521:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:1.0:1775768223.409522:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:0.0:1775768223.409526:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768223.409526:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768223.409528:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768223.409530:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775768223.409532:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:0.0:1775768223.409533:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207888e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294289280:192.168.202.156@tcp:250:kworker.0 02000000:00000001:1.0:1775768223.409534:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775768223.409538:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543f000. 00000100:00000001:0.0:1775768223.409539:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1775768223.409542:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768223.409543:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768223.409546:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768223.409549:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:1.0:1775768223.409551:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207889880 x1862022294289408/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000010:0.0:1775768223.409554:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002a800. 02000000:00000001:0.0:1775768223.409559:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768223.409560:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775768223.409563:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768223.409565:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f11856b8] to pc [ptlrpcd_rcv+-1] req@ffff994207889880 x1862022294289408/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775768223.409567:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768223.409573:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775768223.409581:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000001:1.0:1775768223.409581:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768223.409582:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768223.409586:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775768223.409587:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000200:0.0:1775768223.409589:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294289280, portal 25 00000100:00080000:1.0:1775768223.409589:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768223.409592:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:0.0:1775768223.409593:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768223.409596:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606121984 : -112966103429632 : ffff994207888e00) 00000100:00080000:1.0:1775768223.409597:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5475 00000100:00000040:0.0:1775768223.409604:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207888e00 x1862022294289280/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768248 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775768223.409611:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768223.409618:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:1.0:1775768223.409618:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775768223.409622:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768223.409624:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768223.409628:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000040:1.0:1775768223.409629:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000200:0.0:1775768223.409632:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294289280, offset 0 00000100:00000001:1.0:1775768223.409633:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768223.409638:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768223.409642:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000200:0.0:1775768223.409643:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000020:00000001:1.0:1775768223.409645:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775768223.409647:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:1.0:1775768223.409648:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775768223.409649:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:1.0:1775768223.409652:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000400:00000200:0.0:1775768223.409653:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:1.0:1775768223.409655:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:1.0:1775768223.409659:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000200:0.0:1775768223.409662:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00000001:1.0:1775768223.409662:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768223.409666:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000200:0.0:1775768223.409670:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1775768223.409673:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000200:0.0:1775768223.409675:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000020:00000040:1.0:1775768223.409676:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000400:00000200:0.0:1775768223.409682:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000100:00080000:1.0:1775768223.409682:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775768223.409685:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768223.409688:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:0.0:1775768223.409689:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 02000000:00000001:1.0:1775768223.409691:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768223.409693:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768223.409695:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:0.0:1775768223.409699:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225433:225433:256:4294967295] 192.168.202.156@tcp LPNI seq info [225429:225429:8:4294967295] 00000008:00100000:1.0:1775768223.409699:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775768223.409704:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768223.409707:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768223.409710:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000010:1.0:1775768223.409712:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207888700. 00000020:00000040:1.0:1775768223.409715:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:1.0:1775768223.409717:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768223.409719:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768223.409722:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:0.0:1775768223.409725:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 02000000:00000001:1.0:1775768223.409725:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775768223.409726:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775768223.409731:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543d000. 00000800:00000200:0.0:1775768223.409746:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768223.409751:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000100:00000001:1.0:1775768223.409757:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1775768223.409758:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768223.409764:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000100:00100000:1.0:1775768223.409767:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207888700 x1862022294289536/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:0.1:1775768223.409773:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768223.409779:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000100:00000001:1.0:1775768223.409783:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000100:0.0:1775768223.409787:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000100:00000001:1.0:1775768223.409788:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1775768223.409791:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000100:00000040:1.0:1775768223.409795:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009f2ba301] to pc [ptlrpcd_rcv+-1] req@ffff994207888700 x1862022294289536/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775768223.409798:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:1.0:1775768223.409804:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768223.409808:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000100:00080000:1.0:1775768223.409809:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768223.409814:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775768223.409815:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:0.0:1775768223.409816:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000100:00080000:1.0:1775768223.409816:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000010:0.0:1775768223.409821:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:1.0:1775768223.409821:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775768223.409825:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5475 00000100:00000001:0.0:1775768223.409826:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000400:00000200:1.0:1775768223.409829:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768223.409833:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000200:0.0:1775768223.409834:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207888e00 x1862022294289280/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768248 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768223.409836:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775768223.409839:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768223.409840:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768223.409842:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768223.409843:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775768223.409844:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775768223.409845:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:1.0:1775768223.409845:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775768223.409847:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:1.0:1775768223.409851:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:1.0:1775768223.409855:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:0.0:1775768223.409859:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768223.409860:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768223.409863:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768223.409866:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207888e00 x1862022294289280/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768248 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775768223.409868:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768223.409872:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775768223.409877:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00000001:0.0:1775768223.409882:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768223.409883:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768223.409886:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768223.409888:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768223.409891:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1775768223.409893:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768223.409894:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768223.409898:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768223.409902:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768223.409903:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775768223.409906:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000100:00000001:0.0:1775768223.409907:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4300149565 00000008:00000001:1.0:1775768223.409908:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768223.409911:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768223.409915:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942078a3d80. 00000100:00000010:1.0:1775768223.409915:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff99420788b100. 00000020:00000040:1.0:1775768223.409916:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:1.0:1775768223.409920:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768223.409921:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768223.409922:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1775768223.409923:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 02000000:00000001:1.0:1775768223.409923:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775768223.409924:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775768223.409928:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543e400. 00000100:00000001:0.0:1775768223.409929:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4300149565 00000100:00000001:1.0:1775768223.409931:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768223.409935:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942078a3d20. 00000100:00100000:1.0:1775768223.409936:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff99420788b100 x1862022294289664/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768223.409940:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775768223.409945:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768223.409946:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768223.409947:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768223.409952:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768223.409953:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e26cf977] to pc [ptlrpcd_rcv+-1] req@ffff99420788b100 x1862022294289664/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768223.409956:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000001:1.0:1775768223.409961:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768223.409963:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768223/real 1775768223] req@ffff994207888e00 x1862022294289280/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768223.409964:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768223.409969:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775768223.409971:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775768223.409973:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768223.409978:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000400:00000001:0.0:1775768223.409984:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768223.409985:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768223.409987:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000100:00080000:1.0:1775768223.409989:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5475 00000400:00000200:0.0:1775768223.409991:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768223.409994:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000400:00000200:1.0:1775768223.409995:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768223.409998:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000200:1.0:1775768223.409999:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775768223.410002:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000200:0.0:1775768223.410006:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207888e00 x1862022294289280/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775768223.410006:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768223.410010:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768223.410013:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768223.410016:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775768223.410019:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775768223.410021:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:1.0:1775768223.410023:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775768223.410025:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:1.0:1775768223.410027:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000200:0.0:1775768223.410029:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207888e00 x1862022294289280/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1775768223.410029:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:1.0:1775768223.410035:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768223.410039:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768223.410043:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768223.410045:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:0.0:1775768223.410046:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000040:1.0:1775768223.410047:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:1.0:1775768223.410049:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775768223.410050:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768223.410051:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775768223.410053:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768223.410054:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768223.410055:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768223.410056:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775768223.410058:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000100:00000001:0.0:1775768223.410059:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000008:00000001:1.0:1775768223.410061:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768223.410062:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768223.410063:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775768223.410068:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207889f80. 00000100:00100000:0.0:1775768223.410071:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207888e00 x1862022294289280/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1775768223.410071:0:13471:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:1.0:1775768223.410072:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768223.410074:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768223.410077:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775768223.410079:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775768223.410080:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775768223.410085:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543d400. 00000100:00000001:0.0:1775768223.410090:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768223.410091:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768223.410096:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207888e00 x1862022294289280/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768248 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768223.410099:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207889f80 x1862022294289792/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768223.410116:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768223.410122:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768223.410122:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768223.410125:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768223.410127:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768223.410127:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768223.410132:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768223.410134:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:1.0:1775768223.410134:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ed2274f4] to pc [ptlrpcd_rcv+-1] req@ffff994207889f80 x1862022294289792/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768223.410140:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768223.410149:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768223.410151:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768223.410153:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768223.410174:0:13471:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5485) 00000100:00000040:0.0:1775768223.410190:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207888e00 x1862022294289280/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768248 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768223.410199:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207888e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294289280:192.168.202.156@tcp:250:kworker.0 00000100:00000001:0.0:1775768223.410204:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768223.410206:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768223.410208:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207888e00 x1862022294289280/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768248 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768223.410214:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768223.410217:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768223.410219:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002a800. 02000000:00000001:0.0:1775768223.410222:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768223.410224:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768223.410225:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:0.0:1775768223.410227:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768223.410229:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543dc00. 02000000:00000001:0.0:1775768223.410232:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768223.410234:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768223.410235:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207888e00. 00000100:00000001:0.0:1775768223.410238:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768223.410239:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768223.410242:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768223.410247:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768223.410248:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768223.410250:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775768223.410252:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768223.410253:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768223.410256:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207889880 x1862022294289408/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768223.410263:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768223.410265:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768223.410267:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768223.410269:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768223.410271:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768223.410273:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207889880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294289408:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768223.410277:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768223.410279:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768223.410280:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768223.410282:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768223.410284:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002a800. 02000000:00000001:0.0:1775768223.410286:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768223.410290:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768223.410293:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775768223.410296:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768223.410298:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294289408, portal 10 00000100:00000001:0.0:1775768223.410308:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768223.410309:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606124672 : -112966103426944 : ffff994207889880) 00000100:00000040:0.0:1775768223.410315:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207889880 x1862022294289408/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768248 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768223.410321:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768223.410324:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768223.410326:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775768223.410328:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294289408, offset 0 00000400:00000200:0.0:1775768223.410332:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768223.410335:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768223.410339:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768223.410341:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768223.410344:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768223.410346:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768223.410348:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768223.410351:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225434:225434:256:4294967295] 192.168.202.156@tcp LPNI seq info [225430:225430:8:4294967295] 00000400:00000200:0.0:1775768223.410355:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768223.410361:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768223.410366:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768223.410369:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775768223.410373:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768223.410375:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775768223.410379:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768223.410381:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1775768223.410385:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768223.410386:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775768223.410389:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768223.410392:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768223.410395:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775768223.410396:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775768223.410399:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768223.410402:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207889880 x1862022294289408/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768248 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768223.410409:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768223.410412:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207889880 x1862022294289408/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768248 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768223.410418:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768223.410420:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768223.410423:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768223.410426:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:0.0:1775768223.410429:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768223.410431:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300148243 00000100:00000001:0.0:1775768223.410433:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300149565 00000100:00000001:0.0:1775768223.410435:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300148243 00000100:00000001:0.0:1775768223.410437:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300149565 00000100:00000001:0.0:1775768223.410439:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768223.410441:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768223.410443:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768223.410446:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768223/real 1775768223] req@ffff994207889880 x1862022294289408/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768223.410454:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768223.410455:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768223.410457:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775768223.410459:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768223.410460:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768223.410462:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768223.410465:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207889880 x1862022294289408/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768223.410473:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207889880 x1862022294289408/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768223.410479:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768223.410480:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768223.410482:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768223.410483:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768223.410486:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207889880 x1862022294289408/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768223.410492:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768223.410495:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207889880 x1862022294289408/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768248 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768223.410502:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768223.410505:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768223.410506:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768223.410507:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768223.410509:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768223.410511:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768223.410513:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768223.410516:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768223.410518:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768223.410522:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207889880 x1862022294289408/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768248 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768223.410530:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207889880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294289408:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768223.410533:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768223.410536:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207888700 x1862022294289536/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768223.410542:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768223.410543:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768223.410545:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768223.410546:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768223.410548:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768223.410550:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207888700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294289536:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768223.410553:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768223.410555:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768223.410557:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768223.410558:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768223.410561:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220029800. 02000000:00000001:0.0:1775768223.410563:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768223.410566:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768223.410568:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775768223.410570:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768223.410573:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294289536, portal 10 00000100:00000001:0.0:1775768223.410574:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768223.410576:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606120192 : -112966103431424 : ffff994207888700) 00000100:00000040:0.0:1775768223.410580:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207888700 x1862022294289536/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768248 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768223.410586:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768223.410589:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768223.410590:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775768223.410592:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294289536, offset 0 00000400:00000200:0.0:1775768223.410596:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768223.410599:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768223.410602:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768223.410604:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768223.410607:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768223.410609:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768223.410611:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768223.410614:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225435:225435:256:4294967295] 192.168.202.156@tcp LPNI seq info [225431:225431:8:4294967295] 00000400:00000200:0.0:1775768223.410617:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768223.410623:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768223.410627:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768223.410630:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775768223.410634:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768223.410636:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775768223.410640:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768223.410643:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1775768223.410646:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768223.410648:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775768223.410651:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768223.410654:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768223.410657:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775768223.410658:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775768223.410661:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768223.410664:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207888700 x1862022294289536/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768248 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768223.410671:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768223.410674:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207888700 x1862022294289536/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768248 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768223.410680:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768223.410682:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768223.410684:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768223.410686:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:0.0:1775768223.410689:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768223.410692:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300148235 00000100:00000001:0.0:1775768223.410694:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300149565 00000100:00000001:0.0:1775768223.410696:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300148235 00000100:00000001:0.0:1775768223.410698:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300149565 00000100:00000001:0.0:1775768223.410699:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768223.410702:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768223.410703:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768223.410707:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768223/real 1775768223] req@ffff994207888700 x1862022294289536/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768223.410714:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768223.410715:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768223.410717:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775768223.410719:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768223.410720:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768223.410722:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768223.410725:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207888700 x1862022294289536/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768223.410745:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207888700 x1862022294289536/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768223.410752:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768223.410754:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768223.410755:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768223.410757:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768223.410760:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207888700 x1862022294289536/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768223.410767:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768223.410770:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207888700 x1862022294289536/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768248 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768223.410777:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768223.410780:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768223.410782:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768223.410783:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768223.410785:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768223.410787:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768223.410790:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768223.410793:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768223.410795:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768223.410798:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207888700 x1862022294289536/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768248 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768223.410806:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207888700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294289536:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768223.410809:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768223.410811:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff99420788b100 x1862022294289664/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768223.410816:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768223.410817:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768223.410820:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768223.410821:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768223.410823:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768223.410825:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff99420788b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294289664:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768223.410828:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768223.410830:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768223.410831:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768223.410833:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768223.410836:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220029400. 02000000:00000001:0.0:1775768223.410837:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768223.410841:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768223.410843:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775768223.410846:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768223.410848:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294289664, portal 4 00000100:00000001:0.0:1775768223.410851:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768223.410852:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606130944 : -112966103420672 : ffff99420788b100) 00000100:00000040:0.0:1775768223.410857:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff99420788b100 x1862022294289664/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768248 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768223.410863:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768223.410865:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768223.410867:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775768223.410869:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294289664, offset 0 00000400:00000200:0.0:1775768223.410873:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768223.410876:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768223.410879:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768223.410882:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768223.410884:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768223.410886:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768223.410889:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768223.410892:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225436:225436:256:4294967295] 192.168.202.156@tcp LPNI seq info [225432:225432:8:4294967295] 00000400:00000200:0.0:1775768223.410896:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768223.410902:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768223.410906:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768223.410909:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775768223.410912:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768223.410915:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775768223.410918:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768223.410921:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1775768223.410925:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768223.410927:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775768223.410930:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768223.410933:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768223.410935:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775768223.410937:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775768223.410939:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768223.410942:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff99420788b100 x1862022294289664/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768248 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768223.410955:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768223.410958:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff99420788b100 x1862022294289664/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768248 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768223.410964:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768223.410966:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768223.410968:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768223.410970:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:0.0:1775768223.410973:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768223.410975:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300148258 00000100:00000001:0.0:1775768223.410977:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300149565 00000100:00000001:0.0:1775768223.410979:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300148258 00000100:00000001:0.0:1775768223.410981:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300149565 00000100:00000001:0.0:1775768223.410983:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768223.410984:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768223.410986:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768223.410989:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768223/real 1775768223] req@ffff99420788b100 x1862022294289664/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768223.410996:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768223.410998:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768223.410999:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775768223.411001:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768223.411003:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768223.411005:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768223.411008:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff99420788b100 x1862022294289664/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768223.411016:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff99420788b100 x1862022294289664/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768223.411023:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768223.411024:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768223.411026:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768223.411027:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768223.411030:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff99420788b100 x1862022294289664/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768223.411037:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768223.411040:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff99420788b100 x1862022294289664/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768248 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768223.411047:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768223.411049:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768223.411050:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768223.411051:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768223.411053:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768223.411056:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768223.411058:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768223.411061:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768223.411063:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768223.411067:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff99420788b100 x1862022294289664/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768248 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768223.411074:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff99420788b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294289664:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768223.411078:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768223.411081:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207889f80 x1862022294289792/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768223.411087:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768223.411089:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768223.411091:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768223.411092:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768223.411094:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768223.411097:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207889f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294289792:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768223.411100:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768223.411101:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768223.411103:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768223.411104:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768223.411107:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220028400. 02000000:00000001:0.0:1775768223.411109:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768223.411112:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768223.411114:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775768223.411117:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768223.411119:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294289792, portal 4 00000100:00000001:0.0:1775768223.411121:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768223.411122:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606126464 : -112966103425152 : ffff994207889f80) 00000100:00000040:0.0:1775768223.411126:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207889f80 x1862022294289792/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768248 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768223.411132:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768223.411135:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768223.411136:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775768223.411138:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294289792, offset 0 00000400:00000200:0.0:1775768223.411142:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768223.411145:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768223.411148:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768223.411150:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768223.411152:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768223.411170:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768223.411172:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768223.411175:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225437:225437:256:4294967295] 192.168.202.156@tcp LPNI seq info [225433:225433:8:4294967295] 00000400:00000200:0.0:1775768223.411179:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768223.411185:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768223.411190:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768223.411192:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775768223.411196:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768223.411199:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775768223.411203:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768223.411205:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1775768223.411209:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768223.411211:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775768223.411213:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768223.411217:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768223.411219:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775768223.411221:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775768223.411223:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768223.411226:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207889f80 x1862022294289792/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768248 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768223.411233:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768223.411236:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207889f80 x1862022294289792/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768248 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768223.411242:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768223.411244:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768223.411246:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768223.411249:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:0.0:1775768223.411251:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768223.411253:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300148262 00000100:00000001:0.0:1775768223.411255:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300149565 00000100:00000001:0.0:1775768223.411257:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300148262 00000100:00000001:0.0:1775768223.411259:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300149565 00000100:00000001:0.0:1775768223.411261:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768223.411263:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768223.411264:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768223.411267:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768223/real 1775768223] req@ffff994207889f80 x1862022294289792/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768223.411274:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768223.411275:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768223.411277:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775768223.411279:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768223.411280:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768223.411282:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768223.411285:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207889f80 x1862022294289792/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768223.411293:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207889f80 x1862022294289792/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768223.411299:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768223.411300:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768223.411302:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768223.411303:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768223.411306:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207889f80 x1862022294289792/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768223.411313:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768223.411316:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207889f80 x1862022294289792/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768248 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768223.411322:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768223.411324:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768223.411325:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768223.411327:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768223.411328:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768223.411331:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768223.411333:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768223.411337:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768223.411338:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768223.411342:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207889f80 x1862022294289792/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768248 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768223.411356:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207889f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294289792:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768223.411359:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768223.411361:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768223.411364:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207889880 x1862022294289408/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768248 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768223.411370:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768223.411371:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768223.411373:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002a800. 02000000:00000001:0.0:1775768223.411376:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768223.411377:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768223.411378:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:0.0:1775768223.411380:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768223.411382:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543f000. 02000000:00000001:0.0:1775768223.411384:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768223.411385:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768223.411387:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207889880. 00000100:00000001:0.0:1775768223.411389:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768223.411390:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768223.411392:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768223.411394:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207888700 x1862022294289536/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768248 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768223.411401:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768223.411402:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768223.411403:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220029800. 02000000:00000001:0.0:1775768223.411405:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768223.411406:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768223.411408:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:0.0:1775768223.411409:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768223.411410:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543d000. 02000000:00000001:0.0:1775768223.411412:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768223.411413:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768223.411414:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207888700. 00000100:00000001:0.0:1775768223.411416:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768223.411417:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768223.411419:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768223.411422:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff99420788b100 x1862022294289664/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768248 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768223.411428:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768223.411429:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768223.411430:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220029400. 02000000:00000001:0.0:1775768223.411431:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768223.411432:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768223.411433:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:0.0:1775768223.411435:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768223.411436:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543e400. 02000000:00000001:0.0:1775768223.411438:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768223.411439:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768223.411440:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff99420788b100. 00000100:00000001:0.0:1775768223.411442:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768223.411443:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768223.411444:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768223.411447:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207889f80 x1862022294289792/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768248 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768223.411453:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768223.411454:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768223.411455:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220028400. 02000000:00000001:0.0:1775768223.411457:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768223.411458:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768223.411459:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:0.0:1775768223.411461:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768223.411462:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543d400. 02000000:00000001:0.0:1775768223.411463:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768223.411464:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768223.411465:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207889f80. 00000100:00000001:0.0:1775768223.411467:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768223.411468:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768223.411470:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768223.411473:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768223.411474:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768223.411476:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775768223.411477:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768223.473259:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768223.473263:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768223.473266:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768223.473267:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768223.473270:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768223.473272:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768223.473274:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768223.473275:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768223.473277:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768223.473278:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768223.473332:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768223.473335:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768223.473339:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768223.537248:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768223.537253:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768223.537254:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768223.537258:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768223.537261:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768223.537264:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768223.537266:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768223.537268:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768223.537270:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768224.433251:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768224.433252:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768224.433252:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768224.433257:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768224.433258:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768224.433258:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768224.433260:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768224.433260:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768224.433262:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768224.433266:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775768224.433266:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:0.0:1775768224.433267:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775768224.433272:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768224.433272:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768224.433272:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768224.433277:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768224.433277:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768224.433277:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768224.433287:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768224.433289:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768224.433291:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768224.433292:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768224.433294:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768224.433297:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768224.433298:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768224.433301:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768224.433304:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768224.497249:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768224.497253:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768224.497256:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768224.497257:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768224.497259:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768224.497260:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768224.497262:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768224.497264:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768224.497266:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768224.497266:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768224.497302:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768224.497305:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768224.497308:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768224.561235:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768224.561241:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768224.561242:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768224.561246:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768224.561248:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768224.561251:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768224.561252:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768224.561254:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768224.561256:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768224.753230:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4300149565, at 4300149568 00000100:00000040:0.0:1775768224.753231:0:15036:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4300149565, at 4300149568 00000100:00000010:1.0:1775768224.753244:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3d20. 00000100:00000010:0.0:1775768224.753246:0:15036:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3d80. 00000100:00000001:3.0:1775768225.457234:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768225.457239:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768225.457241:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768225.457245:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768225.457246:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:3.0:1775768225.457246:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768225.457251:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768225.457251:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768225.457252:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768225.457255:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768225.457255:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768225.457257:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768225.457261:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00100000:0.0:1775768225.457262:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775768225.457262:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768225.457263:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768225.457265:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768225.457269:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768225.457269:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768225.457269:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768225.457273:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768225.457275:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768225.457278:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768225.457281:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768225.457281:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768225.457284:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768225.457287:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768225.521251:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768225.521255:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768225.521259:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768225.521260:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768225.521263:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768225.521264:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768225.521267:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768225.521267:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768225.521269:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768225.521270:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768225.521319:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768225.521322:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768225.521326:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768225.585240:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768225.585245:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768225.585247:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768225.585250:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768225.585253:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768225.585256:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768225.585258:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768225.585260:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768225.585261:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768226.481252:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768226.481252:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768226.481258:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768226.481258:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768226.481260:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768226.481260:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768226.481265:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775768226.481265:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775768226.481269:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768226.481272:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768226.481275:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768226.481275:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768226.481277:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768226.481278:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768226.481280:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768226.481280:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768226.481281:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768226.481283:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768226.481303:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768226.481307:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768226.481308:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768226.481311:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775768226.481314:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768226.481316:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768226.481317:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768226.481319:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768226.481321:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768226.545223:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768226.545227:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768226.545231:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768226.545234:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768226.545236:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768226.545238:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768226.545240:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768226.545241:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768226.545243:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768226.545244:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768226.545246:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768226.545247:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768226.545248:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768226.545248:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768226.545249:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768226.545250:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768226.545251:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768226.545252:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768226.545254:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768226.545255:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768226.545256:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768226.545257:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768226.545257:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768226.545258:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768226.545260:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768226.545261:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768226.545263:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768226.545264:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768226.545265:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768226.545265:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768226.545266:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768226.545267:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768226.545268:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768226.545269:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768226.545313:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768226.545317:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768226.545320:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768226.609258:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768226.609265:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768226.609267:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768226.609273:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768226.609277:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768226.609280:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768226.609282:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768226.609285:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768226.609288:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768227.505224:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768227.505224:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768227.505224:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768227.505230:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768227.505230:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768227.505231:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768227.505231:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768227.505233:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768227.505234:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768227.505236:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775768227.505238:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775768227.505238:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768227.505240:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768227.505244:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768227.505245:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768227.505245:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768227.505247:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768227.505248:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768227.505248:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768227.505251:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768227.505254:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768227.505254:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768227.505257:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768227.505259:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768227.505261:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768227.505264:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768227.505264:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768227.569239:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768227.569243:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768227.569247:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768227.569248:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768227.569250:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768227.569252:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768227.569255:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768227.569256:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768227.569259:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768227.569260:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768227.569324:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768227.569327:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768227.569331:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768227.633252:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768227.633259:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768227.633260:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768227.633265:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768227.633268:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768227.633271:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768227.633273:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768227.633275:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768227.633276:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768228.529267:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768228.529267:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768228.529274:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768228.529275:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768228.529278:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768228.529278:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768228.529285:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775768228.529285:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775768228.529290:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768228.529291:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768228.529295:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768228.529296:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768228.529301:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768228.529304:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768228.529315:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768228.529397:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775768228.529398:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768228.529401:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768228.529404:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768228.529405:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:0.0:1775768228.529407:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775768228.529409:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1775768228.529411:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1775768228.529413:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768228.529414:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768228.529419:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775768228.529427:0:13471:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5480 00000400:00000200:2.0:1775768228.529443:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768228.529449:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775768228.529453:0:13471:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1775768228.529467:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768228.529470:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768228.529472:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768228.529473:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768228.529475:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768228.529478:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768228.529519:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768228.529523:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768228.529525:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768228.529527:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775768228.529530:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775768228.529532:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:2.0:1775768228.529535:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775768228.529538:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:2.0:1775768228.529540:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:2.0:1775768228.529542:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:2.0:1775768228.529545:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768228.529547:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768228.529549:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768228.529550:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775768228.529553:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:2.0:1775768228.529556:0:13471:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775768228.529558:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768228.529561:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775768228.529563:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768228.529566:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768228.529568:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775768228.529571:0:13471:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775768228.529578:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfdc00. 00000020:00000040:2.0:1775768228.529582:0:13471:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:2.0:1775768228.529586:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775768228.529587:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768228.529590:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775768228.529592:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775768228.529596:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775768228.529602:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1e400. 00000100:00000001:2.0:1775768228.529608:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768228.529615:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfdc00 x1862022294289920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775768228.529624:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775768228.529629:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768228.529634:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003e25603c] to pc [ptlrpcd_rcv+-1] req@ffff994207dfdc00 x1862022294289920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768228.529662:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768228.529664:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768228.529667:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775768228.529668:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775768228.529670:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775768228.529673:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775768228.529675:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5480 00000400:00000200:2.0:1775768228.529679:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768228.529697:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775768228.529700:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775768228.529702:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768228.529704:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768228.529706:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768228.529708:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775768228.529710:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775768228.529712:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:2.0:1775768228.529714:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775768228.529716:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:2.0:1775768228.529718:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:2.0:1775768228.529721:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000001:0.0:1775768228.529724:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1775768228.529724:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768228.529728:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1775768228.529729:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:2.0:1775768228.529731:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768228.529732:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1775768228.529735:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775768228.529738:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00000040:0.0:1775768228.529741:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfdc00 x1862022294289920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768228.529753:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768228.529755:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768228.529755:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775768228.529761:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768228.529762:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768228.529765:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768228.529766:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768228.529770:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768228.529771:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1775768228.529776:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294289920:192.168.202.156@tcp:250:kworker.0 02000000:00000001:2.0:1775768228.529776:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768228.529780:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775768228.529782:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000008:00100000:2.0:1775768228.529786:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1775768228.529787:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768228.529790:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1775768228.529791:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768228.529793:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1775768228.529794:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768228.529800:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220028400. 00000100:00000010:2.0:1775768228.529800:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dff100. 02000000:00000001:0.0:1775768228.529804:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775768228.529805:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:2.0:1775768228.529810:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775768228.529811:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768228.529815:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768228.529815:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775768228.529818:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775768228.529820:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775768228.529821:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 02000000:00000010:2.0:1775768228.529825:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1e800. 00000400:00000010:0.0:1775768228.529829:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000001:2.0:1775768228.529830:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1775768228.529837:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294289920, portal 25 00000100:00100000:2.0:1775768228.529838:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dff100 x1862022294290048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768228.529842:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768228.529845:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611843584 : -112966097708032 : ffff994207dfdc00) 00000100:00000040:0.0:1775768228.529854:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfdc00 x1862022294289920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768253 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768228.529869:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768228.529873:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1775768228.529873:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768228.529878:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000100:00000040:2.0:1775768228.529881:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3e64ae1] to pc [ptlrpcd_rcv+-1] req@ffff994207dff100 x1862022294290048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1775768228.529882:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775768228.529886:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294289920, offset 0 00000400:00000200:0.0:1775768228.529897:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00000001:2.0:1775768228.529898:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768228.529901:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768228.529905:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000200:0.0:1775768228.529906:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768228.529909:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775768228.529912:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:0.0:1775768228.529915:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00000001:2.0:1775768228.529916:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:0.0:1775768228.529920:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:2.0:1775768228.529923:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5480 00000400:00000200:0.0:1775768228.529927:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768228.529931:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768228.529933:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768228.529938:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000400:00000200:0.0:1775768228.529939:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00080000:2.0:1775768228.529943:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000400:00000200:0.0:1775768228.529946:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225438:225438:256:4294967295] 192.168.202.156@tcp LPNI seq info [225434:225434:8:4294967295] 00000100:00000040:2.0:1775768228.529950:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000200:0.0:1775768228.529955:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768228.529956:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768228.529960:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768228.529962:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775768228.529967:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775768228.529969:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000400:00000200:0.0:1775768228.529971:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000020:00000001:2.0:1775768228.529973:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775768228.529977:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000800:00000200:0.0:1775768228.529981:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000020:00000001:2.0:1775768228.529981:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:2.0:1775768228.529986:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000800:00000010:0.0:1775768228.529988:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000100:00000040:2.0:1775768228.529990:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768228.529994:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768228.529998:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000800:00000200:0.0:1775768228.529999:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:2.0:1775768228.530001:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000800:00000200:0.0:1775768228.530004:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000020:00000040:2.0:1775768228.530005:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:2.0:1775768228.530011:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775768228.530015:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775768228.530015:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 02000000:00000001:2.0:1775768228.530019:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:0.1:1775768228.530020:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 02000000:00000001:2.0:1775768228.530023:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000100:0.0:1775768228.530027:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 02000000:00000001:2.0:1775768228.530028:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1775768228.530030:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000100:00000001:2.0:1775768228.530031:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775768228.530034:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000400:00000200:0.0:1775768228.530037:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000008:00000001:2.0:1775768228.530037:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768228.530042:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768228.530045:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000100:00000010:2.0:1775768228.530048:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfc700. 00000400:00000200:0.0:1775768228.530049:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000020:00000040:2.0:1775768228.530053:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000400:00000010:0.0:1775768228.530054:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 02000000:00000001:2.0:1775768228.530060:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1775768228.530061:0:2399:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:2.0:1775768228.530062:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768228.530065:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000200:0.0:1775768228.530067:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfdc00 x1862022294289920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768253 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1775768228.530067:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775768228.530071:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775768228.530077:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1d400. 00000100:00000001:0.0:1775768228.530081:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1775768228.530082:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768228.530087:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfdc00 x1862022294289920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768253 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768228.530091:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfc700 x1862022294290176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768228.530100:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768228.530103:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768228.530107:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768228.530110:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768228.530110:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768228.530117:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000040:2.0:1775768228.530118:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a176dd1] to pc [ptlrpcd_rcv+-1] req@ffff994207dfc700 x1862022294290176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768228.530121:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4300154685 00000100:00000010:0.0:1775768228.530128:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942078a3d80. 00000100:00000001:2.0:1775768228.530133:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768228.530136:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00080000:2.0:1775768228.530136:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768228.530142:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775768228.530143:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4300154685 00000100:00000001:2.0:1775768228.530144:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775768228.530147:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000010:0.0:1775768228.530148:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942078a31e0. 00000100:00000001:0.0:1775768228.530153:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1775768228.530153:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775768228.530178:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5480 00000100:00000001:0.0:1775768228.530182:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1775768228.530185:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768228.530187:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768228.530190:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000400:00000200:2.0:1775768228.530191:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775768228.530195:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775768228.530199:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768228/real 1775768228] req@ffff994207dfdc00 x1862022294289920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775768228.530203:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768228.530208:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768228.530212:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000001:0.0:1775768228.530214:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1775768228.530215:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000001:0.0:1775768228.530217:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:2.0:1775768228.530220:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000400:00000010:0.0:1775768228.530221:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000020:00000040:2.0:1775768228.530222:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000400:00000200:0.0:1775768228.530224:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000020:00000001:2.0:1775768228.530225:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000010:0.0:1775768228.530227:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000020:00000040:2.0:1775768228.530228:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000100:00000001:0.0:1775768228.530231:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:2.0:1775768228.530232:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:2.0:1775768228.530236:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000200:0.0:1775768228.530237:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfdc00 x1862022294289920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775768228.530244:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768228.530249:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:0.0:1775768228.530253:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfdc00 x1862022294289920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775768228.530253:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768228.530259:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775768228.530264:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00000001:0.0:1775768228.530269:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00080000:2.0:1775768228.530269:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768228.530272:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768228.530273:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768228.530276:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:2.0:1775768228.530276:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1775768228.530279:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1775768228.530281:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1775768228.530286:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfdc00 x1862022294289920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1775768228.530286:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768228.530291:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775768228.530295:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000100:00000001:0.0:1775768228.530298:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1775768228.530300:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768228.530303:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768228.530305:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfdc00 x1862022294289920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768253 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1775768228.530308:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfe300. 00000020:00000040:2.0:1775768228.530313:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00000100:00000001:0.0:1775768228.530319:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1775768228.530320:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775768228.530323:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768228.530324:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768228.530328:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768228.530328:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1775768228.530329:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1775768228.530331:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1775768228.530332:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 02000000:00000001:2.0:1775768228.530334:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768228.530335:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768228.530340:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 02000000:00000010:2.0:1775768228.530341:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1c800. 00000100:00080000:0.0:1775768228.530344:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768228.530347:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1775768228.530347:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768228.530356:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfe300 x1862022294290304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:0.0:1775768228.530357:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfdc00 x1862022294289920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768253 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768228.530385:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294289920:192.168.202.156@tcp:250:kworker.0 00000100:00000001:2.0:1775768228.530385:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775768228.530389:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768228.530392:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768228.530395:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768228.530398:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c661fef] to pc [ptlrpcd_rcv+-1] req@ffff994207dfe300 x1862022294290304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775768228.530400:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfdc00 x1862022294289920/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768253 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768228.530426:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1775768228.530428:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768228.530429:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768228.530433:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220028400. 00000100:00080000:2.0:1775768228.530433:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:0.0:1775768228.530438:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00080000:2.0:1775768228.530438:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000020:00000001:0.0:1775768228.530441:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:2.0:1775768228.530441:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000020:00000040:0.0:1775768228.530443:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000100:00080000:2.0:1775768228.530444:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000020:00000001:0.0:1775768228.530448:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1775768228.530449:0:13471:0:(import.c:484:import_select_connection()) Process entered 02000000:00000010:0.0:1775768228.530452:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1e400. 00000100:00080000:2.0:1775768228.530454:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5480 02000000:00000001:0.0:1775768228.530458:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000400:00000200:2.0:1775768228.530460:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:0.0:1775768228.530461:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768228.530464:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfdc00. 00000400:00000200:2.0:1775768228.530466:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:0.0:1775768228.530468:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00080000:2.0:1775768228.530469:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1775768228.530471:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768228.530474:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768228.530476:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768228.530479:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768228.530481:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768228.530482:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768228.530484:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768228.530484:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775768228.530489:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:0.0:1775768228.530490:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:2.0:1775768228.530492:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000100:00000001:0.0:1775768228.530493:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768228.530495:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000001:2.0:1775768228.530496:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775768228.530499:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000100:00000040:0.0:1775768228.530502:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dff100 x1862022294290048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1775768228.530503:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:2.0:1775768228.530508:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000001:0.0:1775768228.530515:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000040:2.0:1775768228.530516:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768228.530518:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768228.530520:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1775768228.530521:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768228.530524:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1775768228.530525:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000001:0.0:1775768228.530528:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768228.530529:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775768228.530533:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00100000:0.0:1775768228.530534:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294290048:192.168.202.156@tcp:38:kworker.0 00000100:00080000:2.0:1775768228.530539:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768228.530540:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768228.530543:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1775768228.530544:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768228.530545:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768228.530548:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768228.530549:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1775768228.530554:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768228.530557:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768228.530559:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775768228.530561:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 02000000:00000010:0.0:1775768228.530562:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220028400. 00000008:00000001:2.0:1775768228.530563:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768228.530567:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768228.530568:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768228.530575:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000010:2.0:1775768228.530575:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfed80. 00000020:00000040:2.0:1775768228.530580:0:13471:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00000400:00000010:0.0:1775768228.530582:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 02000000:00000001:2.0:1775768228.530585:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000010:0.0:1775768228.530587:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 02000000:00000001:2.0:1775768228.530587:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768228.530591:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000200:0.0:1775768228.530592:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294290048, portal 10 02000000:00000001:2.0:1775768228.530593:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1775768228.530597:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1775768228.530597:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768228.530599:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611848960 : -112966097702656 : ffff994207dff100) 02000000:00000010:2.0:1775768228.530602:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1e400. 00000100:00000040:0.0:1775768228.530607:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dff100 x1862022294290048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768253 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768228.530607:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768228.530616:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfed80 x1862022294290432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768228.530619:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768228.530625:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768228.530630:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000001:2.0:1775768228.530631:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000200:0.0:1775768228.530633:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294290048, offset 0 00000100:00000001:2.0:1775768228.530635:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768228.530639:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00000040:2.0:1775768228.530642:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb9b9eae] to pc [ptlrpcd_rcv+-1] req@ffff994207dfed80 x1862022294290432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775768228.530646:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768228.530653:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768228.530658:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1775768228.530659:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768228.530662:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768228.530665:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000100:00000040:2.0:1775768228.530665:0:13471:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5491) 00000400:00000200:0.0:1775768228.530671:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768228.530674:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225439:225439:256:4294967295] 192.168.202.156@tcp LPNI seq info [225435:225435:8:4294967295] 00000400:00000200:0.0:1775768228.530678:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768228.530695:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768228.530699:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768228.530703:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775768228.530708:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768228.530711:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1775768228.530714:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768228.530717:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000100:0.0:1775768228.530721:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768228.530723:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775768228.530725:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768228.530728:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768228.530731:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775768228.530732:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775768228.530735:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768228.530737:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dff100 x1862022294290048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768253 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768228.530745:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768228.530749:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dff100 x1862022294290048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768253 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768228.530755:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768228.530756:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768228.530759:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768228.530763:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:0.0:1775768228.530766:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768228.530768:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300153363 00000100:00000001:0.0:1775768228.530770:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300154685 00000100:00000001:0.0:1775768228.530772:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300153363 00000100:00000001:0.0:1775768228.530774:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300154685 00000100:00000001:0.0:1775768228.530776:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768228.530778:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768228.530780:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768228.530784:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768228/real 1775768228] req@ffff994207dff100 x1862022294290048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768228.530791:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768228.530793:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768228.530794:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775768228.530796:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768228.530798:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768228.530800:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768228.530803:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dff100 x1862022294290048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768228.530811:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dff100 x1862022294290048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768228.530817:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768228.530819:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768228.530821:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768228.530822:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768228.530825:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dff100 x1862022294290048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768228.530831:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768228.530836:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dff100 x1862022294290048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768253 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768228.530842:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768228.530845:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768228.530847:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768228.530849:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768228.530851:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768228.530854:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768228.530856:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768228.530860:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768228.530863:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768228.530868:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dff100 x1862022294290048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768253 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768228.530875:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294290048:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768228.530879:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768228.530882:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfc700 x1862022294290176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768228.530890:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768228.530891:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768228.530893:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768228.530894:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768228.530897:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768228.530899:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294290176:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768228.530903:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768228.530904:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768228.530906:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768228.530908:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768228.530911:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220029400. 02000000:00000001:0.0:1775768228.530913:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768228.530916:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768228.530919:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775768228.530923:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768228.530926:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294290176, portal 10 00000100:00000001:0.0:1775768228.530928:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768228.530929:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611838208 : -112966097713408 : ffff994207dfc700) 00000100:00000040:0.0:1775768228.530935:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfc700 x1862022294290176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768253 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768228.530941:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768228.530944:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768228.530946:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775768228.530949:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294290176, offset 0 00000400:00000200:0.0:1775768228.530954:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768228.530958:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768228.530962:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768228.530966:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768228.530968:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768228.530971:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768228.530973:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768228.530977:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225440:225440:256:4294967295] 192.168.202.156@tcp LPNI seq info [225436:225436:8:4294967295] 00000400:00000200:0.0:1775768228.530980:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768228.530986:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768228.530992:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768228.530996:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775768228.531000:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768228.531002:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1775768228.531007:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768228.531010:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000100:0.0:1775768228.531013:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768228.531015:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775768228.531018:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768228.531022:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768228.531025:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775768228.531027:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775768228.531029:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768228.531033:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfc700 x1862022294290176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768253 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768228.531041:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768228.531045:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfc700 x1862022294290176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768253 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768228.531051:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768228.531053:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768228.531056:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768228.531059:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:0.0:1775768228.531062:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768228.531064:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300153355 00000100:00000001:0.0:1775768228.531067:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300154685 00000100:00000001:0.0:1775768228.531069:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300153355 00000100:00000001:0.0:1775768228.531071:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300154685 00000100:00000001:0.0:1775768228.531073:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768228.531076:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768228.531078:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768228.531081:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768228/real 1775768228] req@ffff994207dfc700 x1862022294290176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768228.531087:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768228.531088:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768228.531090:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775768228.531092:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768228.531093:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768228.531095:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768228.531098:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfc700 x1862022294290176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768228.531103:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfc700 x1862022294290176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768228.531108:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768228.531109:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768228.531110:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768228.531110:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768228.531112:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfc700 x1862022294290176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768228.531116:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768228.531118:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfc700 x1862022294290176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768253 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768228.531121:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768228.531122:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768228.531123:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768228.531124:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768228.531126:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768228.531127:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768228.531129:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768228.531132:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768228.531133:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768228.531135:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfc700 x1862022294290176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768253 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768228.531139:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294290176:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768228.531141:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768228.531143:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfe300 x1862022294290304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768228.531146:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768228.531147:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768228.531148:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768228.531149:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768228.531150:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768228.531152:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294290304:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768228.531174:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768228.531176:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768228.531177:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768228.531179:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768228.531182:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220029800. 02000000:00000001:0.0:1775768228.531185:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768228.531188:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768228.531191:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775768228.531194:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768228.531198:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294290304, portal 4 00000100:00000001:0.0:1775768228.531200:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768228.531201:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611845376 : -112966097706240 : ffff994207dfe300) 00000100:00000040:0.0:1775768228.531207:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfe300 x1862022294290304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768253 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768228.531213:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768228.531216:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768228.531219:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775768228.531222:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294290304, offset 0 00000400:00000200:0.0:1775768228.531226:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768228.531230:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768228.531234:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768228.531237:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768228.531240:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768228.531243:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768228.531245:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768228.531248:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225441:225441:256:4294967295] 192.168.202.156@tcp LPNI seq info [225437:225437:8:4294967295] 00000400:00000200:0.0:1775768228.531252:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768228.531258:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768228.531264:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768228.531267:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775768228.531273:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768228.531276:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1775768228.531280:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768228.531282:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000100:0.0:1775768228.531286:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768228.531288:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775768228.531290:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768228.531293:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768228.531295:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775768228.531296:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775768228.531299:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768228.531301:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfe300 x1862022294290304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768253 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768228.531306:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768228.531308:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfe300 x1862022294290304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768253 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768228.531313:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768228.531314:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768228.531316:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768228.531318:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:0.0:1775768228.531320:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768228.531322:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300153379 00000100:00000001:0.0:1775768228.531324:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300154685 00000100:00000001:0.0:1775768228.531326:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300153379 00000100:00000001:0.0:1775768228.531328:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300154685 00000100:00000001:0.0:1775768228.531330:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768228.531332:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768228.531333:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768228.531336:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768228/real 1775768228] req@ffff994207dfe300 x1862022294290304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768228.531343:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768228.531344:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768228.531346:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775768228.531348:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768228.531349:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768228.531351:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768228.531354:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfe300 x1862022294290304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768228.531360:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfe300 x1862022294290304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768228.531366:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768228.531368:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768228.531369:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768228.531371:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768228.531374:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfe300 x1862022294290304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768228.531380:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768228.531383:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfe300 x1862022294290304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768253 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768228.531390:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768228.531391:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768228.531392:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768228.531393:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768228.531394:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768228.531397:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768228.531398:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768228.531400:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768228.531401:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768228.531404:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfe300 x1862022294290304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768253 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768228.531408:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294290304:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768228.531410:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768228.531411:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768228.531413:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dff100 x1862022294290048/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768253 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768228.531417:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768228.531418:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768228.531419:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220028400. 02000000:00000001:0.0:1775768228.531420:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768228.531421:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768228.531422:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:0.0:1775768228.531423:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768228.531424:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1e800. 02000000:00000001:0.0:1775768228.531426:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768228.531427:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768228.531428:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dff100. 00000100:00000001:0.0:1775768228.531429:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768228.531430:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768228.531431:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768228.531433:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfc700 x1862022294290176/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768253 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768228.531437:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768228.531437:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768228.531438:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220029400. 02000000:00000001:0.0:1775768228.531439:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768228.531440:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768228.531440:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:0.0:1775768228.531441:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768228.531442:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1d400. 02000000:00000001:0.0:1775768228.531444:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768228.531444:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768228.531445:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfc700. 00000100:00000001:0.0:1775768228.531446:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768228.531447:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768228.531447:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768228.531449:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfe300 x1862022294290304/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768253 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768228.531452:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768228.531453:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768228.531453:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220029800. 02000000:00000001:0.0:1775768228.531454:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768228.531455:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768228.531455:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:0.0:1775768228.531456:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768228.531457:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1c800. 02000000:00000001:0.0:1775768228.531458:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768228.531458:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768228.531459:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfe300. 00000100:00000001:0.0:1775768228.531460:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768228.531461:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768228.531462:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768228.531464:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768228.531465:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768228.531466:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775768228.531467:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768228.531468:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768228.531470:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfed80 x1862022294290432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768228.531474:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768228.531475:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768228.531476:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768228.531477:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768228.531478:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768228.531480:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294290432:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768228.531481:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768228.531482:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768228.531483:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768228.531484:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768228.531486:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220029800. 02000000:00000001:0.0:1775768228.531487:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768228.531489:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768228.531490:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000400:00000010:0.0:1775768228.531492:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768228.531494:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294290432, portal 4 00000100:00000001:0.0:1775768228.531495:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768228.531496:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611848064 : -112966097703552 : ffff994207dfed80) 00000100:00000040:0.0:1775768228.531499:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfed80 x1862022294290432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768253 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768228.531503:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768228.531504:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768228.531505:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775768228.531507:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294290432, offset 0 00000400:00000200:0.0:1775768228.531509:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768228.531511:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768228.531513:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768228.531515:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768228.531516:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768228.531518:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768228.531519:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768228.531521:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225442:225442:256:4294967295] 192.168.202.156@tcp LPNI seq info [225438:225438:8:4294967295] 00000400:00000200:0.0:1775768228.531524:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768228.531527:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768228.531530:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768228.531531:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000800:00000200:0.0:1775768228.531534:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768228.531535:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1775768228.531537:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768228.531539:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 1 secs later) 00000800:00000100:0.0:1775768228.531541:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768228.531542:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:0.0:1775768228.531544:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768228.531546:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768228.531547:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:0.0:1775768228.531548:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:0.0:1775768228.531550:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768228.531552:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfed80 x1862022294290432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768253 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768228.531556:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768228.531558:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfed80 x1862022294290432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768253 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768228.531561:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768228.531562:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768228.531563:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768228.531565:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:0.0:1775768228.531567:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768228.531568:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300153382 00000100:00000001:0.0:1775768228.531569:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300154685 00000100:00000001:0.0:1775768228.531571:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300153382 00000100:00000001:0.0:1775768228.531573:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300154685 00000100:00000001:0.0:1775768228.531574:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768228.531576:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768228.531577:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768228.531579:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768228/real 1775768228] req@ffff994207dfed80 x1862022294290432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768228.531582:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768228.531583:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768228.531584:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775768228.531585:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768228.531586:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768228.531587:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768228.531588:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfed80 x1862022294290432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768228.531593:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfed80 x1862022294290432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768228.531596:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768228.531597:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768228.531598:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768228.531599:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768228.531600:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfed80 x1862022294290432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768228.531604:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768228.531606:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfed80 x1862022294290432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768253 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768228.531609:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768228.531610:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768228.531611:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768228.531612:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768228.531613:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768228.531615:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768228.531616:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768228.531618:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768228.531620:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768228.531622:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfed80 x1862022294290432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768253 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768228.531626:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294290432:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768228.531628:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768228.531629:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768228.531630:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfed80 x1862022294290432/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768253 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768228.531633:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768228.531634:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768228.531635:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220029800. 02000000:00000001:0.0:1775768228.531636:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768228.531637:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768228.531637:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:0.0:1775768228.531638:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768228.531639:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1e400. 02000000:00000001:0.0:1775768228.531640:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768228.531641:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768228.531642:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfed80. 00000100:00000001:0.0:1775768228.531644:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768228.531645:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768228.531646:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768228.531648:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768228.531650:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768228.531651:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775768228.531652:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768228.593252:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768228.593257:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768228.593262:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768228.593263:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768228.593265:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768228.593266:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768228.593268:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768228.593269:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768228.593272:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768228.593273:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768228.593330:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768228.593333:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768228.593336:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768228.657230:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768228.657234:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768228.657235:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768228.657239:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768228.657242:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768228.657245:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768228.657247:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768228.657248:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768228.657250:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768229.553241:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768229.553247:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768229.553248:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768229.553249:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768229.553253:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768229.553256:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768229.553256:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:0.0:1775768229.553259:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768229.553259:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768229.553261:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768229.553264:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768229.553264:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768229.553266:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768229.553267:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768229.553268:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768229.553271:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00100000:1.0:1775768229.553272:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768229.553274:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768229.553275:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768229.553279:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768229.553280:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768229.553280:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768229.553282:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768229.553285:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768229.553286:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768229.553288:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768229.553291:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768229.617270:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768229.617277:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768229.617283:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768229.617284:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768229.617288:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768229.617289:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768229.617292:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768229.617293:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768229.617296:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768229.617297:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768229.617356:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768229.617360:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768229.617364:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768229.681249:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768229.681254:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768229.681256:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768229.681260:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768229.681263:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768229.681266:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768229.681267:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768229.681269:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768229.681271:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768229.873255:0:15036:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4300154685, at 4300154688 00000100:00000040:1.0:1775768229.873255:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4300154685, at 4300154688 00000100:00000010:1.0:1775768229.873269:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a31e0. 00000100:00000010:0.0:1775768229.873272:0:15036:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3d80. 00000100:00000001:3.0:1775768230.577209:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768230.577216:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768230.577218:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768230.577221:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:3.0:1775768230.577223:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768230.577225:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768230.577226:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768230.577227:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768230.577227:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1775768230.577229:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768230.577231:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768230.577233:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768230.577233:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768230.577234:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768230.577234:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768230.577237:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768230.577238:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768230.577243:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768230.577243:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768230.577243:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768230.577246:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768230.577248:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768230.577249:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768230.577252:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768230.577254:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768230.577256:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768230.577258:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768230.641238:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768230.641243:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768230.641247:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768230.641247:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768230.641250:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768230.641250:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768230.641252:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768230.641253:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768230.641255:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768230.641256:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768230.641292:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768230.641295:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768230.641297:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768230.705228:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768230.705235:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768230.705237:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768230.705242:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768230.705246:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768230.705250:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768230.705252:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768230.705254:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768230.705256:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768231.601278:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768231.601284:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768231.601287:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768231.601291:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768231.601294:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768231.601294:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768231.601297:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768231.601301:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768231.601301:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768231.601303:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768231.601306:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775768231.601309:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768231.601311:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768231.601314:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768231.601318:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768231.601320:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768231.601322:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768231.601324:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768231.601421:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768231.601424:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768231.601425:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768231.601427:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:0.0:1775768231.601428:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768231.601430:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768231.601431:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768231.601432:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768231.601433:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768231.665292:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768231.665300:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768231.665303:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768231.665306:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768231.665311:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768231.665314:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768231.665315:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768231.665317:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768231.665320:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768231.665321:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768231.665324:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768231.665324:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768231.665327:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768231.665328:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768231.665331:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768231.665332:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768231.665418:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768231.665424:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768231.665429:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768231.729285:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768231.729292:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768231.729294:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768231.729300:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768231.729303:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768231.729308:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768231.729310:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768231.729312:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768231.729315:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768232.625316:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768232.625321:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768232.625323:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768232.625325:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768232.625328:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768232.625330:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768232.625332:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768232.625332:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768232.625339:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768232.625342:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:0.0:1775768232.625343:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775768232.625347:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768232.625349:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768232.625350:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768232.625356:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768232.625357:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768232.625358:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768232.625361:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768232.625364:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768232.625365:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768232.625373:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768232.625373:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768232.625378:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768232.625382:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768232.625383:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768232.625386:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768232.625389:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768232.689246:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768232.689251:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768232.689256:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768232.689257:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768232.689260:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768232.689261:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768232.689264:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768232.689265:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768232.689268:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768232.689269:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768232.689305:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768232.689309:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768232.689317:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768232.753287:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768232.753296:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768232.753298:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768232.753304:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768232.753308:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768232.753318:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768232.753321:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768232.753323:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768232.753325:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768233.649293:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768233.649301:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768233.649303:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768233.649306:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:3.0:1775768233.649309:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768233.649311:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768233.649315:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768233.649316:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:2.0:1775768233.649320:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768233.649321:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768233.649326:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768233.649327:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768233.649330:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768233.649330:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768233.649338:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768233.649338:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768233.649341:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768233.649347:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768233.649357:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768233.649363:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775768233.649366:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775768233.649370:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768233.649373:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768233.649374:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:0.0:1775768233.649380:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768233.649383:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768233.649387:0:13471:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5486 00000100:00100000:0.0:1775768233.649389:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775768233.649395:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768233.649398:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768233.649400:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768233.649404:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1775768233.649405:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768233.649412:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768233.649420:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775768233.649424:0:13471:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775768233.649428:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768233.649432:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768233.649435:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768233.649437:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775768233.649440:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775768233.649442:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:1.0:1775768233.649444:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775768233.649447:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:1.0:1775768233.649449:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:1.0:1775768233.649452:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:1.0:1775768233.649455:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768233.649456:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768233.649458:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768233.649460:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775768233.649463:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:1.0:1775768233.649466:0:13471:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775768233.649468:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768233.649471:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775768233.649474:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768233.649477:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768233.649479:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775768233.649481:0:13471:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775768233.649489:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207889f80. 00000020:00000040:1.0:1775768233.649493:0:13471:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:1.0:1775768233.649496:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768233.649498:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768233.649501:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775768233.649504:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775768233.649508:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775768233.649517:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543e400. 00000100:00000001:1.0:1775768233.649524:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768233.649532:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207889f80 x1862022294290560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775768233.649542:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775768233.649546:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768233.649551:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ed2274f4] to pc [ptlrpcd_rcv+-1] req@ffff994207889f80 x1862022294290560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768233.649572:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768233.649575:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768233.649578:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775768233.649580:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775768233.649582:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768233.649585:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775768233.649588:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5486 00000400:00000200:1.0:1775768233.649593:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768233.649597:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775768233.649599:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1775768233.649603:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1775768233.649603:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768233.649609:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1775768233.649610:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1775768233.649613:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768233.649614:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1775768233.649623:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775768233.649627:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000040:0.0:1775768233.649646:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207889f80 x1862022294290560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1775768233.649647:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:1.0:1775768233.649652:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775768233.649662:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:1.0:1775768233.649668:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:1.0:1775768233.649678:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000001:0.0:1775768233.649688:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768233.649691:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768233.649692:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768233.649698:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1775768233.649700:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:1.0:1775768233.649701:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000001:0.0:1775768233.649706:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768233.649706:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775768233.649712:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 02000000:00000001:0.0:1775768233.649713:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768233.649720:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00100000:0.0:1775768233.649724:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207889f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294290560:192.168.202.156@tcp:250:kworker.0 00000100:00000001:1.0:1775768233.649727:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768233.649734:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1775768233.649736:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768233.649741:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768233.649742:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768233.649750:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768233.649750:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768233.649757:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:0.0:1775768233.649758:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:1.0:1775768233.649761:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775768233.649766:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768233.649767:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220029800. 00000100:00000001:1.0:1775768233.649769:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768233.649773:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775768233.649778:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff99420788b100. 00000020:00000040:1.0:1775768233.649782:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000400:00000200:0.0:1775768233.649785:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:1.0:1775768233.649793:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768233.649794:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775768233.649797:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbcc60. 00000100:00000001:1.0:1775768233.649797:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775768233.649803:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775768233.649806:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775768233.649810:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 02000000:00000010:1.0:1775768233.649812:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543d000. 00000100:00000001:1.0:1775768233.649817:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1775768233.649820:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294290560, portal 25 00000100:00100000:1.0:1775768233.649825:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff99420788b100 x1862022294290688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768233.649826:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768233.649829:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606126464 : -112966103425152 : ffff994207889f80) 00000100:00000001:1.0:1775768233.649842:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:0.0:1775768233.649843:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207889f80 x1862022294290560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768258 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768233.649847:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768233.649860:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e26cf977] to pc [ptlrpcd_rcv+-1] req@ffff99420788b100 x1862022294290688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768233.649864:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768233.649875:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768233.649878:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014880. 00000100:00000200:0.0:1775768233.649882:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294290560, offset 0 00000400:00000200:0.0:1775768233.649895:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00000001:1.0:1775768233.649896:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768233.649901:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768233.649904:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775768233.649907:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:0.0:1775768233.649908:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00080000:1.0:1775768233.649911:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768233.649916:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:0.0:1775768233.649921:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00080000:1.0:1775768233.649922:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5486 00000400:00000200:0.0:1775768233.649931:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768233.649937:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768233.649941:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768233.649944:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768233.649949:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225443:225443:256:4294967295] 192.168.202.156@tcp LPNI seq info [225439:225439:8:4294967295] 00000400:00000200:0.0:1775768233.649954:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768233.649965:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000400:00000200:1.0:1775768233.649975:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000800:00000200:0.0:1775768233.649976:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768233.649985:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6100. 00000400:00000200:1.0:1775768233.649986:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775768233.649992:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775768233.649997:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000800:00000200:0.0:1775768233.650000:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:1.0:1775768233.650001:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768233.650011:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000800:00000200:0.1:1775768233.650013:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:1.0:1775768233.650018:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775768233.650022:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775768233.650024:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:1.0:1775768233.650026:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775768233.650027:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:1.0:1775768233.650029:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:1.0:1775768233.650031:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000001:0.0:1775768233.650035:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768233.650036:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768233.650043:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1775768233.650046:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000040:1.0:1775768233.650048:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768233.650051:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4300159805 00000100:00000001:1.0:1775768233.650052:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775768233.650057:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00000010:0.0:1775768233.650059:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942078a3d80. 00000100:00080000:1.0:1775768233.650064:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775768233.650068:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768233.650071:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 02000000:00000001:1.0:1775768233.650071:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775768233.650076:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768233.650080:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4300159805 02000000:00000001:1.0:1775768233.650080:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768233.650086:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775768233.650089:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775768233.650092:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768233.650095:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768233.650096:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942078a3960. 00000100:00000010:1.0:1775768233.650101:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207888700. 00000100:00000001:0.0:1775768233.650102:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000020:00000040:1.0:1775768233.650106:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000100:00000001:0.0:1775768233.650110:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768233.650111:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768233.650116:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768233.650117:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768233.650120:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1775768233.650122:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768233.650123:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775768233.650126:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768233.650128:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:1.0:1775768233.650133:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543f000. 00000100:00000001:1.0:1775768233.650138:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768233.650139:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768233.650142:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00100000:1.0:1775768233.650146:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207888700 x1862022294290816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768233.650148:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775768233.650152:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768233.650186:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1775768233.650193:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:0.0:1775768233.650195:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff99420788b100 x1862022294290688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768233.650197:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768233.650210:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009f2ba301] to pc [ptlrpcd_rcv+-1] req@ffff994207888700 x1862022294290816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768233.650217:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768233.650221:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768233.650229:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768233.650233:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768233.650233:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768233.650242:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768233.650243:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768233.650251:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1775768233.650252:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff99420788b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294290688:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768233.650254:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775768233.650258:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768233.650268:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:0.0:1775768233.650270:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768233.650275:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:1.0:1775768233.650275:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5486 02000000:00000001:0.0:1775768233.650280:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768233.650284:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:1.0:1775768233.650286:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000010:0.0:1775768233.650290:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220029400. 00000400:00000200:1.0:1775768233.650291:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 02000000:00000001:0.0:1775768233.650295:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768233.650296:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775768233.650305:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000200:0.0:1775768233.650306:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768233.650312:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:0.0:1775768233.650314:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbc8f0. 00000100:00000040:1.0:1775768233.650317:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000010:0.0:1775768233.650320:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9942050142a8. 00000100:00000001:1.0:1775768233.650321:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000200:0.0:1775768233.650328:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294290688, portal 10 00000020:00000001:1.0:1775768233.650328:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775768233.650330:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000100:00000001:0.0:1775768233.650336:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000020:00000001:1.0:1775768233.650337:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:0.0:1775768233.650340:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606130944 : -112966103420672 : ffff99420788b100) 00000020:00000040:1.0:1775768233.650342:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:1.0:1775768233.650347:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:1.0:1775768233.650350:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:0.0:1775768233.650354:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff99420788b100 x1862022294290688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768258 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775768233.650360:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768233.650367:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768233.650373:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768233.650382:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:0.0:1775768233.650383:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000040:1.0:1775768233.650387:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00000040:0.0:1775768233.650390:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768233.650396:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014f68. 00000100:00080000:1.0:1775768233.650399:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000200:0.0:1775768233.650401:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294290688, offset 0 00000100:00000001:1.0:1775768233.650406:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768233.650410:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:0.0:1775768233.650416:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 02000000:00000001:1.0:1775768233.650418:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1775768233.650425:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:1.0:1775768233.650425:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768233.650434:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775768233.650437:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000400:00000200:0.0:1775768233.650438:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000008:00000001:1.0:1775768233.650448:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768233.650449:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1775768233.650454:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768233.650455:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768233.650462:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000100:00000010:1.0:1775768233.650464:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207889880. 00000400:00000200:0.0:1775768233.650467:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000020:00000040:1.0:1775768233.650472:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00000400:00000200:0.0:1775768233.650477:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225444:225444:255:4294967295] 192.168.202.156@tcp LPNI seq info [225440:225440:7:4294967295] 02000000:00000001:1.0:1775768233.650479:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768233.650483:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768233.650489:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:0.0:1775768233.650490:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:1.0:1775768233.650492:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775768233.650496:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768233.650506:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 02000000:00000010:1.0:1775768233.650507:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543dc00. 00000100:00000001:1.0:1775768233.650516:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1775768233.650517:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00100000:1.0:1775768233.650528:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207889880 x1862022294290944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775768233.650536:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775768233.650538:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1775768233.650541:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6000. 00000100:00000040:1.0:1775768233.650542:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f11856b8] to pc [ptlrpcd_rcv+-1] req@ffff994207889880 x1862022294290944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1775768233.650556:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768233.650568:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:1.0:1775768233.650569:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768233.650577:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775768233.650578:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768233.650589:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000040:0.0:1775768233.650590:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000100:00000001:1.0:1775768233.650597:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:0.0:1775768233.650599:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00080000:1.0:1775768233.650601:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768233.650603:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300158483 00000100:00000001:1.0:1775768233.650608:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:0.0:1775768233.650610:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300159805 00000100:00000001:0.0:1775768233.650614:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300158483 00000100:00080000:1.0:1775768233.650615:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5486 00000100:00000001:0.0:1775768233.650621:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300159805 00000100:00000001:0.0:1775768233.650626:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1775768233.650627:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768233.650645:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768233.650649:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768233.650652:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:0.0:1775768233.650654:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1775768233.650659:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1775768233.650662:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768233.650666:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000040:1.0:1775768233.650668:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768233.650672:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768233.650674:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1775768233.650676:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768233.650683:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768233.650683:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768233.650688:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000040:0.0:1775768233.650693:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207888700 x1862022294290816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1775768233.650694:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775768233.650701:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:1.0:1775768233.650706:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775768233.650711:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:1.0:1775768233.650721:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:1.0:1775768233.650728:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000001:0.0:1775768233.650731:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768233.650735:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768233.650740:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:1.0:1775768233.650741:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 02000000:00000001:0.0:1775768233.650744:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768233.650745:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1775768233.650752:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768233.650753:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768233.650761:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00100000:0.0:1775768233.650763:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207888700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294290816:192.168.202.156@tcp:38:kworker.0 00000020:00000040:1.0:1775768233.650767:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00000001:0.0:1775768233.650775:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768233.650778:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:1.0:1775768233.650779:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 02000000:00000001:0.0:1775768233.650782:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768233.650786:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1775768233.650786:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768233.650791:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000010:0.0:1775768233.650792:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220028400. 02000000:00000001:0.0:1775768233.650797:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768233.650797:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768233.650804:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768233.650810:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768233.650811:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000010:0.0:1775768233.650816:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbc948. 00000008:00100000:1.0:1775768233.650817:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000400:00000010:0.0:1775768233.650825:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9942050145d8. 00000008:00000001:1.0:1775768233.650825:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768233.650832:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1775768233.650834:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294290816, portal 10 00000100:00000010:1.0:1775768233.650837:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207888e00. 00000100:00000001:0.0:1775768233.650839:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768233.650842:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606120192 : -112966103431424 : ffff994207888700) 00000020:00000040:1.0:1775768233.650844:0:13471:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:1.0:1775768233.650852:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000040:0.0:1775768233.650854:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207888700 x1862022294290816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768258 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1775768233.650856:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768233.650866:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775768233.650869:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775768233.650872:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768233.650877:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768233.650882:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 02000000:00000010:1.0:1775768233.650882:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543c800. 00000400:00000010:0.0:1775768233.650889:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014330. 00000100:00000001:1.0:1775768233.650891:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1775768233.650893:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294290816, offset 0 00000100:00100000:1.0:1775768233.650900:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207888e00 x1862022294291072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1775768233.650904:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768233.650918:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768233.650927:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:0.0:1775768233.650932:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00000001:1.0:1775768233.650936:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768233.650940:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768233.650947:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000100:00000040:1.0:1775768233.650948:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001812f966] to pc [ptlrpcd_rcv+-1] req@ffff994207888e00 x1862022294291072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775768233.650954:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768233.650962:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768233.650972:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225445:225445:254:4294967295] 192.168.202.156@tcp LPNI seq info [225441:225441:6:4294967295] 00000100:00000001:1.0:1775768233.650980:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768233.650986:0:13471:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5496) 00000400:00000200:0.0:1775768233.650996:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768233.651004:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768233.651011:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768233.651016:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6a00. 00000800:00000200:0.0:1775768233.651021:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768233.651025:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:0.0:1775768233.651028:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768233.651032:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:0.0:1775768233.651035:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768233.651037:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300158475 00000100:00000001:0.0:1775768233.651039:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300159805 00000100:00000001:0.0:1775768233.651042:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300158475 00000100:00000001:0.0:1775768233.651044:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300159805 00000100:00000001:0.0:1775768233.651047:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768233.651049:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768233.651051:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768233.651055:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207889880 x1862022294290944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768233.651063:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768233.651064:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768233.651067:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768233.651070:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768233.651073:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768233.651076:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207889880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294290944:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768233.651080:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768233.651082:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768233.651084:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768233.651086:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768233.651089:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002a800. 02000000:00000001:0.0:1775768233.651091:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768233.651095:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768233.651098:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbccb8. 00000400:00000010:0.0:1775768233.651102:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014a18. 00000100:00000200:0.0:1775768233.651106:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294290944, portal 4 00000100:00000001:0.0:1775768233.651107:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768233.651109:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606124672 : -112966103426944 : ffff994207889880) 00000100:00000040:0.0:1775768233.651115:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207889880 x1862022294290944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768258 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768233.651135:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768233.651138:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768233.651141:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014220. 00000100:00000200:0.0:1775768233.651144:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294290944, offset 0 00000400:00000200:0.0:1775768233.651148:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768233.651152:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768233.651188:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768233.651192:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768233.651194:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768233.651196:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768233.651199:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768233.651202:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225446:225446:253:4294967295] 192.168.202.156@tcp LPNI seq info [225442:225442:5:4294967295] 00000400:00000200:0.0:1775768233.651206:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768233.651211:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768233.651215:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768233.651219:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6600. 00000800:00000200:0.0:1775768233.651223:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768233.651226:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:0.0:1775768233.651229:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768233.651232:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:0.0:1775768233.651235:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768233.651237:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300158499 00000100:00000001:0.0:1775768233.651239:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300159805 00000100:00000001:0.0:1775768233.651242:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300158499 00000100:00000001:0.0:1775768233.651244:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300159805 00000100:00000001:0.0:1775768233.651246:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768233.651248:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768233.651250:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768233.651252:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768233.651257:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768233.651258:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1775768233.651261:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775768233.651262:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768233.651264:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768233.651268:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207888e00 x1862022294291072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768233.651275:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768233.651277:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768233.651279:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768233.651282:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768233.651284:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768233.651287:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207888e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294291072:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768233.651290:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768233.651292:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768233.651294:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768233.651296:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768233.651300:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002bc00. 02000000:00000001:0.0:1775768233.651301:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768233.651304:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768233.651307:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbc268. 00000400:00000010:0.0:1775768233.651310:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014ee0. 00000100:00000200:0.0:1775768233.651313:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294291072, portal 4 00000100:00000001:0.0:1775768233.651315:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768233.651316:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606121984 : -112966103429632 : ffff994207888e00) 00000100:00000040:0.0:1775768233.651322:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207888e00 x1862022294291072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768258 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768233.651329:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768233.651331:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768233.651333:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014aa0. 00000100:00000200:0.0:1775768233.651335:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294291072, offset 0 00000400:00000200:0.0:1775768233.651339:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768233.651342:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768233.651345:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768233.651347:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768233.651350:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768233.651352:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768233.651355:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768233.651358:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225447:225447:252:4294967295] 192.168.202.156@tcp LPNI seq info [225443:225443:4:4294967295] 00000400:00000200:0.0:1775768233.651361:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768233.651366:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768233.651371:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768233.651374:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6c00. 00000800:00000200:0.0:1775768233.651378:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768233.651381:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:0.0:1775768233.651384:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768233.651387:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:0.0:1775768233.651390:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768233.651392:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300158502 00000100:00000001:0.0:1775768233.651393:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300159805 00000100:00000001:0.0:1775768233.651396:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300158502 00000100:00000001:0.0:1775768233.651398:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300159805 00000100:00000001:0.0:1775768233.651400:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768233.651402:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768233.651404:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768233.651406:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768233.651408:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768233.651409:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1775768233.651411:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775768233.651412:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768233.651414:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768233.651415:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768233.713263:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768233.713269:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768233.713273:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768233.713274:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768233.713277:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768233.713278:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768233.713280:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768233.713281:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768233.713283:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768233.713284:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768233.713339:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768233.713344:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768233.713348:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768233.777276:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768233.777284:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768233.777285:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768233.777290:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768233.777293:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768233.777298:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768233.777300:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768233.777302:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768233.777304:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768234.673283:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768234.673283:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768234.673290:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768234.673292:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768234.673293:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768234.673295:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768234.673300:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775768234.673300:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768234.673305:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768234.673306:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768234.673311:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768234.673312:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768234.673315:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768234.673318:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768234.673320:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768234.673324:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768234.673328:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768234.673331:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768234.673347:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768234.673352:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768234.673354:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768234.673358:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775768234.673361:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768234.673364:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768234.673366:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768234.673368:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768234.673370:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768234.737292:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768234.737300:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768234.737309:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768234.737310:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768234.737314:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768234.737315:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768234.737319:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768234.737321:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768234.737325:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768234.737326:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768234.737393:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768234.737398:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768234.737402:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768234.801301:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768234.801310:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768234.801312:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768234.801320:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768234.801324:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768234.801329:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768234.801332:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768234.801335:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768234.801338:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768234.993253:0:15036:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4300159805, at 4300159808 00000100:00000040:1.0:1775768234.993266:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4300159805, at 4300159808 00000100:00000010:0.0:1775768234.993273:0:15036:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3d80. 00000100:00000010:1.0:1775768234.993278:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3960. 00000100:00000001:0.0:1775768235.697302:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768235.697313:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768235.697316:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768235.697324:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775768235.697331:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768235.697337:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768235.697341:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768235.697344:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768235.697344:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768235.697348:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768235.697350:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768235.697354:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768235.697356:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768235.697359:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768235.697360:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768235.697366:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:1.0:1775768235.697370:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768235.697370:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768235.697374:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768235.697378:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768235.697379:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768235.697381:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768235.697384:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768235.697386:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768235.697391:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768235.697395:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768235.697399:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768235.761276:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768235.761282:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768235.761293:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768235.761294:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768235.761297:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768235.761298:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768235.761300:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768235.761301:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768235.761303:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768235.761304:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768235.761395:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768235.761400:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768235.761405:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768235.825315:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768235.825322:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768235.825324:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768235.825329:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768235.825332:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768235.825337:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768235.825339:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768235.825341:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768235.825343:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768236.721260:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768236.721266:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768236.721268:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768236.721272:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768236.721275:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768236.721278:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768236.721278:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768236.721280:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768236.721282:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768236.721283:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768236.721286:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768236.721292:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768236.721296:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768236.721298:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768236.721301:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768236.721304:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768236.721306:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768236.721309:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768236.721366:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768236.721369:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768236.721370:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768236.721373:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775768236.721374:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768236.721376:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768236.721377:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768236.721378:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768236.721379:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:3.0:1775768236.721445:0:2390:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.156:988 00000400:00000100:3.0:1775768236.721492:0:2390:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.156@tcp at host 192.168.202.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:3.0:1775768236.721502:0:2390:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775768236.721505:0:2390:0:(router.c:1750:lnet_notify()) 192.168.202.56@tcp notifying 192.168.202.156@tcp: down 00000400:00000200:3.0:1775768236.721510:0:2390:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775768236.721514:0:2390:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:3.0:1775768236.721518:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000800:00000010:3.0:1775768236.721522:0:2390:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6100. 00000400:00000200:3.0:1775768236.721527:0:2390:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1775768236.721531:0:2390:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1775768236.721537:0:2390:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014880 00000400:00000010:3.0:1775768236.721539:0:2390:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014880. 00000100:00000001:3.0:1775768236.721544:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775768236.721552:0:2390:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff994207889f80 x1862022294290560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768258 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768236.721583:0:2390:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775768236.721587:0:2390:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207889f80 x1862022294290560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768258 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768236.721594:0:2390:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768236.721596:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1775768236.721612:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000800:00000010:3.0:1775768236.721614:0:2390:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6000. 00000400:00000200:3.0:1775768236.721617:0:2390:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1775768236.721620:0:2390:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1775768236.721623:0:2390:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014f68 00000400:00000010:3.0:1775768236.721624:0:2390:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014f68. 00000100:00000001:3.0:1775768236.721626:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775768236.721630:0:2390:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff99420788b100 x1862022294290688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768258 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768236.721638:0:2390:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775768236.721642:0:2390:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff99420788b100 x1862022294290688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768258 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768236.721648:0:2390:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768236.721650:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1775768236.721655:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000800:00000010:3.0:1775768236.721657:0:2390:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6a00. 00000400:00000200:3.0:1775768236.721659:0:2390:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1775768236.721662:0:2390:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1775768236.721664:0:2390:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014330 00000400:00000010:3.0:1775768236.721666:0:2390:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014330. 00000100:00000001:3.0:1775768236.721668:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775768236.721672:0:2390:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff994207888700 x1862022294290816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768258 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768236.721684:0:2390:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775768236.721686:0:2390:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207888700 x1862022294290816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768258 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768236.721692:0:2390:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768236.721694:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1775768236.721706:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000800:00000010:3.0:1775768236.721708:0:2390:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6600. 00000400:00000200:3.0:1775768236.721710:0:2390:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1775768236.721713:0:2390:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1775768236.721715:0:2390:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014220 00000400:00000010:3.0:1775768236.721717:0:2390:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014220. 00000100:00000001:3.0:1775768236.721719:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775768236.721723:0:2390:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff994207889880 x1862022294290944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768258 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768236.721732:0:2390:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775768236.721735:0:2390:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207889880 x1862022294290944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768258 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768236.721742:0:2390:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768236.721744:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768236.721746:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000100:3.0:1775768236.721749:0:2390:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000100:00000001:0.0:1775768236.721752:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000800:00000010:3.0:1775768236.721753:0:2390:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6c00. 00000100:00000001:0.0:1775768236.721755:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000400:00000200:3.0:1775768236.721756:0:2390:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1775768236.721761:0:2390:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1775768236.721765:0:2390:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014aa0 00000100:00100000:0.0:1775768236.721766:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768233/real 1775768236] req@ffff994207888700 x1862022294290816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1775768236.721769:0:2390:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014aa0. 00000100:00000001:3.0:1775768236.721773:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775768236.721779:0:2390:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff994207888e00 x1862022294291072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768258 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768236.721786:0:2390:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775768236.721789:0:2390:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207888e00 x1862022294291072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768258 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768236.721794:0:2390:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1775768236.721796:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1775768236.721796:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000001:0.0:1775768236.721799:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768236.721804:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbc948. 00000400:00000200:0.0:1775768236.721809:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9942050145d8 00000400:00000010:0.0:1775768236.721811:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff9942050145d8. 00000100:00000001:0.0:1775768236.721814:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768236.721819:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207888700 x1862022294290816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768236.721832:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207888700 x1862022294290816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768236.721839:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768236.721840:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768236.721843:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768236.721845:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768236.721860:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207888700 x1862022294290816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768236.721867:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768236.721872:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207888700 x1862022294290816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768258 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768236.721880:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768236.721883:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768236.721884:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768236.721885:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768236.721887:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768236.721891:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768236.721894:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768236.721898:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768236.721901:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768236.721904:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207888700 x1862022294290816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768258 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768236.721912:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207888700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294290816:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768236.721917:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768236.721919:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768233/real 1775768236] req@ffff994207889880 x1862022294290944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768236.721932:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768236.721933:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768236.721935:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbccb8. 00000400:00000200:0.0:1775768236.721938:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014a18 00000400:00000010:0.0:1775768236.721940:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014a18. 00000100:00000001:0.0:1775768236.721941:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768236.721944:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207889880 x1862022294290944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768236.721951:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207889880 x1862022294290944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768236.721957:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768236.721959:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768236.721960:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768236.721962:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768236.721964:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207889880 x1862022294290944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768236.721971:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768236.721974:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207889880 x1862022294290944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768258 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768236.721979:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768236.721981:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768236.721983:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768236.721984:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768236.721985:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768236.721988:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768236.721990:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768236.721994:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768236.721996:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768236.722000:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207889880 x1862022294290944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768258 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768236.722007:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207889880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294290944:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768236.722011:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768236.722014:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768233/real 1775768236] req@ffff99420788b100 x1862022294290688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768236.722020:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768236.722021:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768236.722022:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbc8f0. 00000400:00000200:0.0:1775768236.722025:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9942050142a8 00000400:00000010:0.0:1775768236.722026:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff9942050142a8. 00000100:00000001:0.0:1775768236.722028:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768236.722030:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff99420788b100 x1862022294290688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768236.722036:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff99420788b100 x1862022294290688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768236.722042:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768236.722043:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768236.722044:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768236.722046:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768236.722049:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff99420788b100 x1862022294290688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768236.722054:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768236.722056:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff99420788b100 x1862022294290688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768258 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768236.722062:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768236.722064:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768236.722066:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768236.722067:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768236.722068:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768236.722071:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768236.722073:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768236.722075:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768236.722077:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768236.722080:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff99420788b100 x1862022294290688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768258 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768236.722086:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff99420788b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294290688:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768236.722090:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768236.722092:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768233/real 1775768236] req@ffff994207889f80 x1862022294290560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768236.722098:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768236.722099:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768236.722101:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbcc60. 00000400:00000200:0.0:1775768236.722102:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768236.722104:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768236.722105:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768236.722108:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207889f80 x1862022294290560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768236.722115:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207889f80 x1862022294290560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768236.722120:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768236.722121:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768236.722122:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768236.722124:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768236.722127:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207889f80 x1862022294290560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768236.722142:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768236.722146:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207889f80 x1862022294290560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768258 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768236.722151:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768236.722153:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768236.722177:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768236.722179:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768236.722180:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768236.722181:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768236.722184:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768236.722187:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768236.722190:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768236.722194:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207889f80 x1862022294290560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768258 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768236.722200:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207889f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294290560:192.168.202.156@tcp:250:kworker.0 00000100:00000001:0.0:1775768236.722204:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768236.722206:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768236.722208:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207888700 x1862022294290816/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768258 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768236.722214:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768236.722217:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768236.722222:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220028400. 02000000:00000001:0.0:1775768236.722225:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768236.722228:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768236.722230:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:0.0:1775768236.722232:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768236.722234:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543f000. 02000000:00000001:0.0:1775768236.722237:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768236.722240:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768236.722242:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207888700. 00000100:00000001:0.0:1775768236.722246:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768236.722247:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768236.722249:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768236.722252:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207889880 x1862022294290944/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768258 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768236.722262:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768236.722264:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768236.722265:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002a800. 02000000:00000001:0.0:1775768236.722268:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768236.722269:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768236.722270:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:0.0:1775768236.722272:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768236.722273:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543dc00. 02000000:00000001:0.0:1775768236.722275:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768236.722276:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768236.722277:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207889880. 00000100:00000001:0.0:1775768236.722279:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768236.722280:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768236.722747:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768236.722755:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff99420788b100 x1862022294290688/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768258 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768236.722761:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768236.722763:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768236.722765:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220029400. 02000000:00000001:0.0:1775768236.722768:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768236.722769:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768236.722770:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:0.0:1775768236.722772:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768236.722773:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543d000. 02000000:00000001:0.0:1775768236.722776:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768236.722778:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768236.722780:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff99420788b100. 00000100:00000001:0.0:1775768236.722782:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768236.722836:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768236.722839:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768236.722843:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207889f80 x1862022294290560/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768258 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768236.722915:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768236.722917:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768236.722919:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220029800. 02000000:00000001:0.0:1775768236.722922:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768236.722923:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768236.722924:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:0.0:1775768236.722926:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768236.722927:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543e400. 02000000:00000001:0.0:1775768236.722929:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768236.722930:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768236.722931:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207889f80. 00000100:00000001:0.0:1775768236.722933:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768236.722934:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768236.722937:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768236.722938:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775768236.722940:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768236.722941:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768236.722945:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768233/real 1775768236] req@ffff994207888e00 x1862022294291072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768236.722954:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768236.722955:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768236.722957:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbc268. 00000400:00000200:0.0:1775768236.722959:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014ee0 00000400:00000010:0.0:1775768236.722960:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014ee0. 00000100:00000001:0.0:1775768236.722963:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768236.722965:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207888e00 x1862022294291072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768236.722973:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207888e00 x1862022294291072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768236.722978:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768236.722979:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768236.722981:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768236.722983:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768236.722986:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207888e00 x1862022294291072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768236.722992:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768236.722995:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207888e00 x1862022294291072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768258 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768236.723001:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768236.723003:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768236.723005:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768236.723005:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768236.723007:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768236.723011:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768236.723013:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768236.723017:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768236.723019:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768236.723022:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207888e00 x1862022294291072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768258 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768236.723027:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207888e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294291072:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768236.723031:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768236.723033:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768236.723036:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207888e00 x1862022294291072/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768258 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768236.723041:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768236.723042:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768236.723044:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002bc00. 02000000:00000001:0.0:1775768236.723046:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768236.723048:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768236.723049:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:0.0:1775768236.723056:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768236.723057:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543c800. 02000000:00000001:0.0:1775768236.723060:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768236.723061:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768236.723062:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207888e00. 00000100:00000001:0.0:1775768236.723064:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768236.723065:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768236.723067:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768236.723075:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768236.723077:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768236.723079:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775768236.723080:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768236.785261:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768236.785267:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768236.785272:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768236.785275:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768236.785277:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768236.785281:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768236.785284:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768236.785286:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768236.785288:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768236.785290:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768236.785293:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768236.785295:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768236.785296:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768236.785298:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768236.785299:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768236.785301:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768236.785302:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768236.785304:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768236.785307:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768236.785309:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768236.785310:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768236.785312:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768236.785321:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768236.785323:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768236.785325:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768236.785330:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768236.785334:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768236.785335:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768236.785337:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768236.785338:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768236.785340:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768236.785341:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768236.785342:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768236.785344:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768236.785411:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768236.785415:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768236.785418:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768236.849253:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768236.849258:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768236.849259:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768236.849263:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768236.849266:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768236.849269:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768236.849271:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768236.849273:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768236.849274:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768237.745221:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768237.745226:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768237.745228:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768237.745228:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:1.0:1775768237.745232:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775768237.745233:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768237.745236:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768237.745237:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768237.745240:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775768237.745242:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768237.745245:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768237.745246:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768237.745248:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768237.745250:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768237.745252:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768237.745253:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768237.745256:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768237.745259:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768237.745287:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768237.745291:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768237.745292:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768237.745295:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775768237.745297:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768237.745300:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768237.745301:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768237.745303:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768237.745305:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768237.809231:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768237.809236:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768237.809239:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768237.809240:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768237.809242:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768237.809243:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768237.809245:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768237.809246:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768237.809248:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768237.809248:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768237.809306:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768237.809310:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768237.809313:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768237.873260:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768237.873267:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768237.873269:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768237.873283:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768237.873287:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768237.873290:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768237.873292:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768237.873295:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768237.873298:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768238.769274:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768238.769285:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768238.769288:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768238.769295:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768238.769301:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768238.769306:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768238.769309:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768238.769312:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1775768238.769314:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775768238.769317:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768238.769318:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768238.769320:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775768238.769320:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768238.769323:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768238.769325:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1775768238.769326:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768238.769329:0:36:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768238.769330:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:2.0:1775768238.769330:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768238.769338:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768238.769339:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775768238.769339:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768238.769345:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768238.769347:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00100000:1.0:1775768238.769348:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768238.769349:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1775768238.769351:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5491 00000100:00000001:2.0:1775768238.769357:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:3.0:1775768238.769359:0:13471:0:(rw.c:1481:ll_writepages()) Process entered 00000100:00000001:1.0:1775768238.769360:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000080:00000001:3.0:1775768238.769363:0:13471:0:(file.c:5250:cl_sync_file_range()) Process entered 00000400:00000200:0.0:1775768238.769367:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:3.0:1775768238.769367:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00000001:1.0:1775768238.769368:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:3.0:1775768238.769373:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff99420693e000. 00000100:00000001:1.0:1775768238.769375:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000400:00000200:0.0:1775768238.769378:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:1.0:1775768238.769381:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1775768238.769382:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000020:00000010:3.0:1775768238.769382:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dce800. 00000020:00000010:3.0:1775768238.769390:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210dcf800. 00000100:00000040:0.0:1775768238.769394:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768238.769394:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1775768238.769396:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207981e40. 00000100:00000001:0.0:1775768238.769406:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000008:00000010:3.0:1775768238.769410:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994208b50000. 00000100:00000040:0.0:1775768238.769411:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768238.769415:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00020000:00000010:3.0:1775768238.769415:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994218000300. 00008000:00000010:3.0:1775768238.769420:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994207e08e20. 00000020:00000001:0.0:1775768238.769421:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768238.769425:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:0.0:1775768238.769428:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000080:00000010:3.0:1775768238.769429:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99420541b1f8. 00000020:00000040:0.0:1775768238.769432:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000080:00000010:3.0:1775768238.769434:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077c0248. 00000020:00000001:0.0:1775768238.769436:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000010:3.0:1775768238.769440:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcf000. 00000020:00000001:0.0:1775768238.769441:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:0.0:1775768238.769446:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000008:00000010:3.0:1775768238.769447:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942086f8420. 00000100:00000001:0.0:1775768238.769449:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00020000:00000010:3.0:1775768238.769452:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994207e8e400. 00000100:00000040:0.0:1775768238.769453:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00008000:00000010:3.0:1775768238.769459:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994209270a90. 00000080:00000010:3.0:1775768238.769463:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942090e33c0. 00000020:00000001:3.0:1775768238.769466:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107590086664 : -112966119464952 : ffff99420693e008) 00000100:00000001:0.0:1775768238.769467:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00001000:3.0:1775768238.769472:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff99420693e000 00000020:00000040:0.0:1775768238.769473:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:0.0:1775768238.769478:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768238.769481:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768238.769483:0:13471:0:(cl_object.c:151:cl_object_top()) ffff99420791a5f8 -> ffff99420791a5f8 02000000:00000001:0.0:1775768238.769487:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000020:00000001:3.0:1775768238.769488:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 02000000:00000001:0.0:1775768238.769494:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000001:3.0:1775768238.769494:0:13471:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:3.0:1775768238.769498:0:13471:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x10176:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 02000000:00000001:0.0:1775768238.769499:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768238.769505:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000020:00000001:3.0:1775768238.769505:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000100:00000001:0.0:1775768238.769508:0:36:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768238.769508:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:3.0:1775768238.769511:0:13471:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768238.769517:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fc000. 00020000:00000002:3.0:1775768238.769517:0:13471:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x10176:0x0]io ffff9942077c02a8 type 6 ignore/verify layout 1/0 00000020:00000040:0.0:1775768238.769524:0:36:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 00020000:00000002:3.0:1775768238.769524:0:13471:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207dfb6c8) owner 0000000000000000/ffff994220080000 00020000:00000001:3.0:1775768238.769530:0:13471:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 02000000:00000001:0.0:1775768238.769532:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00020000:00000001:3.0:1775768238.769534:0:13471:0:(lov_io.c:495:lov_io_slice_init()) Process entered 02000000:00000001:0.0:1775768238.769535:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768238.769538:0:13471:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00000100:00000001:0.0:1775768238.769540:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00020000:00000001:3.0:1775768238.769541:0:13471:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768238.769545:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00020000:00000001:3.0:1775768238.769545:0:13471:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:3.0:1775768238.769548:0:13471:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:3.0:1775768238.769551:0:13471:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768238.769552:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768238.769553:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:3.0:1775768238.769557:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:3.0:1775768238.769559:0:13471:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00020000:00000002:3.0:1775768238.769561:0:13471:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207dfb6c8) owner 0000000000000000/ffff994220080000 00000020:00000001:3.0:1775768238.769564:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768238.769565:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002bc00. 00000020:00000001:3.0:1775768238.769567:0:13471:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:3.0:1775768238.769570:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00020000:00000001:3.0:1775768238.769572:0:13471:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00000100:00000001:0.0:1775768238.769576:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00020000:00200000:3.0:1775768238.769577:0:13471:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00000002:3.0:1775768238.769583:0:13471:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:3.0:1775768238.769597:0:13471:0:(lov_io.c:155:lov_sub_get()) Process entered 00020000:00000001:3.0:1775768238.769599:0:13471:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000100:00100000:0.0:1775768238.769601:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fc000 x1862022294291200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000001:3.0:1775768238.769602:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:3.0:1775768238.769606:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff99420693ecf0. 00000020:00000010:3.0:1775768238.769612:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcf200. 00000020:00000010:3.0:1775768238.769618:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210dce000. 00000100:00000001:0.0:1775768238.769621:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000020:00000010:3.0:1775768238.769622:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207981900. 00000008:00000010:3.0:1775768238.769628:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994208b51650. 00000100:00000001:0.0:1775768238.769630:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:3.0:1775768238.769633:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994218000000. 00000100:00000040:0.0:1775768238.769638:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008bcb9792] to pc [ptlrpcd_rcv+-1] req@ffff9942099fc000 x1862022294291200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00008000:00000010:3.0:1775768238.769638:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994207e0ade8. 00000080:00000010:3.0:1775768238.769643:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99420541b0a8. 00000080:00000010:3.0:1775768238.769648:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077c1da8. 00000020:00000010:3.0:1775768238.769656:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcea00. 00000008:00000010:3.0:1775768238.769659:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942086f8160. 00020000:00000010:3.0:1775768238.769663:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994207e8ed00. 00008000:00000010:3.0:1775768238.769666:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994209270ef0. 00000080:00000010:3.0:1775768238.769669:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942090e3960. 00000020:00000001:3.0:1775768238.769672:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107590089976 : -112966119461640 : ffff99420693ecf8) 00000020:00001000:3.0:1775768238.769674:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff99420693ecf0 00000020:00000001:3.0:1775768238.769678:0:13471:0:(cl_object.c:151:cl_object_top()) ffff99420791ae90 -> ffff99420791a5f8 00000020:00000001:3.0:1775768238.769680:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:3.0:1775768238.769684:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000100:00000001:0.0:1775768238.769685:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768238.769686:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:3.0:1775768238.769689:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768238.769691:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00020000:00000001:3.0:1775768238.769693:0:13471:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768238.769697:0:13471:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107612435560 : -112966097116056 : ffff994207e8e468) 00000100:00080000:0.0:1775768238.769698:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775768238.769701:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000020:00000001:3.0:1775768238.769703:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000100:00080000:0.0:1775768238.769704:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000020:00000001:3.0:1775768238.769706:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768238.769709:0:36:0:(import.c:484:import_select_connection()) Process entered 00020000:00000001:3.0:1775768238.769710:0:13471:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768238.769714:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5491 00000020:00000001:3.0:1775768238.769715:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768238.769719:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00000400:00000200:0.0:1775768238.769722:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:3.0:1775768238.769723:0:13471:0:(lov_io.c:1108:lov_io_lock()) Process entered 00020000:00000001:3.0:1775768238.769727:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000400:00000200:0.0:1775768238.769729:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000020:00000001:3.0:1775768238.769730:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:3.0:1775768238.769733:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000100:00080000:0.0:1775768238.769735:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000020:00000001:3.0:1775768238.769737:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768238.769741:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000001:3.0:1775768238.769742:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768238.769746:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768238.769747:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00020000:00000001:3.0:1775768238.769748:0:13471:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768238.769751:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000001:3.0:1775768238.769751:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:3.0:1775768238.769752:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768238.769757:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775768238.769757:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768238.769760:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00000020:00000001:0.0:1775768238.769761:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768238.769764:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00020000:00000001:3.0:1775768238.769764:0:13471:0:(lov_io.c:1114:lov_io_start()) Process entered 00000020:00000001:0.0:1775768238.769767:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00020000:00000001:3.0:1775768238.769767:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775768238.769770:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00000020:00000040:0.0:1775768238.769771:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:0.0:1775768238.769776:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000008:00000001:3.0:1775768238.769776:0:13471:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000020:00000001:0.0:1775768238.769780:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000008:00000001:3.0:1775768238.769782:0:13471:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000100:00000040:0.0:1775768238.769784:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768238.769788:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768238.769790:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768238.769793:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000008:00000001:3.0:1775768238.769793:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000020:00000040:0.0:1775768238.769795:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000008:00000020:3.0:1775768238.769797:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000100:00080000:0.0:1775768238.769800:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000008:00000001:3.0:1775768238.769801:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768238.769806:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1775768238.769807:0:13471:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 02000000:00000001:0.0:1775768238.769809:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000008:00000001:3.0:1775768238.769811:0:13471:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000008:00000001:3.0:1775768238.769815:0:13471:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107754940544 : -112965954611072 : ffff994210675880) 02000000:00000001:0.0:1775768238.769819:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00000020:3.0:1775768238.769821:0:13471:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994210675880 ready 0|-|- wr 1|-|+ rd 0|- 10 in flight 02000000:00000001:0.0:1775768238.769825:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1775768238.769827:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000100:00000001:0.0:1775768238.769830:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00000020:3.0:1775768238.769830:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:3.0:1775768238.769833:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1775768238.769837:0:13471:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000008:00100000:0.0:1775768238.769838:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000020:3.0:1775768238.769840:0:13471:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994218232fc0 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000008:00000001:0.0:1775768238.769845:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1775768238.769848:0:13471:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768238.769849:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1775768238.769855:0:13471:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768238.769857:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768238.769859:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ff800. 00020000:00000001:3.0:1775768238.769859:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768238.769863:0:13471:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1775768238.769865:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:3.0:1775768238.769866:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768238.769871:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000020:00000001:3.0:1775768238.769871:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 02000000:00000001:0.0:1775768238.769874:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768238.769876:0:13471:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00000100:00000001:0.0:1775768238.769877:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00020000:00000001:3.0:1775768238.769878:0:13471:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 02000000:00000001:0.0:1775768238.769880:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000020:00000001:3.0:1775768238.769881:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000001:3.0:1775768238.769884:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 02000000:00000001:0.0:1775768238.769886:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768238.769886:0:13471:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768238.769892:0:13471:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000020:00000001:3.0:1775768238.769895:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 02000000:00000010:0.0:1775768238.769896:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220029800. 00000020:00000001:3.0:1775768238.769897:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00020000:00000001:3.0:1775768238.769902:0:13471:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00000100:00000001:0.0:1775768238.769903:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768238.769904:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775768238.769907:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000020:00000001:3.0:1775768238.769908:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:3.0:1775768238.769909:0:13471:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768238.769911:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768238.769913:0:13471:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000020:00000001:3.0:1775768238.769915:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:3.0:1775768238.769917:0:13471:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000100:00100000:0.0:1775768238.769918:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ff800 x1862022294291328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000001:3.0:1775768238.769920:0:13471:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000001:3.0:1775768238.769922:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:3.0:1775768238.769924:0:13471:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00020000:00000001:3.0:1775768238.769927:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775768238.769929:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000001:3.0:1775768238.769932:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000100:00000001:0.0:1775768238.769933:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00020000:00000001:3.0:1775768238.769935:0:13471:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768238.769938:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768238.769938:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768238.769943:0:13471:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000020:00000001:3.0:1775768238.769945:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000100:00000040:0.0:1775768238.769947:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000aa4e8b71] to pc [ptlrpcd_rcv+-1] req@ffff9942099ff800 x1862022294291328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775768238.769948:0:13471:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768238.769952:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:3.0:1775768238.769955:0:13471:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:3.0:1775768238.769959:0:13471:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:3.0:1775768238.769960:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:3.0:1775768238.769963:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:3.0:1775768238.769966:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff99420693ecf0 00000020:00000010:3.0:1775768238.769974:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210dce000. 00000100:00000001:0.0:1775768238.769975:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768238.769980:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000010:3.0:1775768238.769984:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207981900. 00000100:00080000:0.0:1775768238.769985:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775768238.769988:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768238.769990:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768238.769994:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768238.769998:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5491 00000008:00000010:3.0:1775768238.769998:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994208b51650. 00000400:00000200:0.0:1775768238.770005:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000010:3.0:1775768238.770008:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994218000000. 00000400:00000200:0.0:1775768238.770010:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00008000:00000010:3.0:1775768238.770014:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994207e0ade8. 00000100:00080000:0.0:1775768238.770015:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768238.770020:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000080:00000010:3.0:1775768238.770022:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99420541b0a8. 00000100:00000001:0.0:1775768238.770023:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000080:00000010:3.0:1775768238.770028:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077c1da8. 00000100:00000040:0.0:1775768238.770029:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768238.770034:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000010:3.0:1775768238.770036:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcf200. 00000020:00000001:0.0:1775768238.770044:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768238.770046:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000008:00000010:3.0:1775768238.770047:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942086f8160. 00000020:00000001:0.0:1775768238.770048:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768238.770052:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00020000:00000010:3.0:1775768238.770054:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994207e8ed00. 00000020:00000001:0.0:1775768238.770055:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:0.0:1775768238.770059:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00008000:00000010:3.0:1775768238.770060:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994209270ef0. 00000100:00000040:0.0:1775768238.770065:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000080:00000010:3.0:1775768238.770066:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942090e3960. 00000100:00000001:0.0:1775768238.770069:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000010:3.0:1775768238.770071:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcea00. 00000100:00000040:0.0:1775768238.770072:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768238.770076:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000010:3.0:1775768238.770077:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff99420693ecf0. 00000020:00000040:0.0:1775768238.770080:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:0.0:1775768238.770084:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768238.770086:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768238.770089:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768238.770092:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768238.770094:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768238.770096:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775768238.770099:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775768238.770101:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768238.770103:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768238.770108:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fd500. 00000020:00000040:0.0:1775768238.770109:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:0.0:1775768238.770112:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768238.770114:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768238.770116:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768238.770117:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768238.770119:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768238.770123:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220029400. 00000100:00000001:0.0:1775768238.770126:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768238.770130:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fd500 x1862022294291456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768238.770138:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768238.770140:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768238.770143:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f246bb2b] to pc [ptlrpcd_rcv+-1] req@ffff9942099fd500 x1862022294291456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768238.770150:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768238.770153:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768238.770206:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775768238.770208:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768238.770209:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768238.770212:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768238.770215:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5491 00000400:00000200:0.0:1775768238.770219:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768238.770221:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768238.770224:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768238.770228:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768238.770230:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768238.770233:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768238.770235:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768238.770237:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768238.770239:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:0.0:1775768238.770240:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768238.770243:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:0.0:1775768238.770246:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:0.0:1775768238.770248:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:0.0:1775768238.770252:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768238.770255:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768238.770256:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768238.770258:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768238.770260:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:0.0:1775768238.770263:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768238.770266:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768238.770268:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768238.770271:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768238.770273:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768238.770275:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775768238.770277:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:0.0:1775768238.770279:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768238.770281:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768238.770285:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ff100. 00020000:00000001:3.0:1775768238.770287:0:13471:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00000020:00000040:0.0:1775768238.770288:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00020000:00000001:3.0:1775768238.770292:0:13471:0:(lov_io.c:694:lov_io_fini()) Process leaving 02000000:00000001:0.0:1775768238.770293:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768238.770295:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:3.0:1775768238.770296:0:13471:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000100:00000001:0.0:1775768238.770297:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768238.770300:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000080:00200000:3.0:1775768238.770300:0:13471:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x10176:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 02000000:00000001:0.0:1775768238.770303:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:3.0:1775768238.770306:0:13471:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 02000000:00000010:0.0:1775768238.770309:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002a800. 00000020:00000001:3.0:1775768238.770309:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:3.0:1775768238.770313:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff99420693e000 00000100:00000001:0.0:1775768238.770314:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1775768238.770317:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210dcf800. 00000100:00100000:0.0:1775768238.770321:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ff100 x1862022294291584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000010:3.0:1775768238.770322:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207981e40. 00000008:00000010:3.0:1775768238.770327:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994208b50000. 00020000:00000010:3.0:1775768238.770332:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994218000300. 00000100:00000001:0.0:1775768238.770335:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00008000:00000010:3.0:1775768238.770337:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994207e08e20. 00000100:00000001:0.0:1775768238.770338:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:3.0:1775768238.770342:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99420541b1f8. 00000100:00000040:0.0:1775768238.770344:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5e24158] to pc [ptlrpcd_rcv+-1] req@ffff9942099ff100 x1862022294291584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000010:3.0:1775768238.770346:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077c0248. 00000020:00000010:3.0:1775768238.770350:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dce800. 00000008:00000010:3.0:1775768238.770355:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942086f8420. 00000100:00000001:0.0:1775768238.770359:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:3.0:1775768238.770360:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994207e8e400. 00000100:00080000:0.0:1775768238.770363:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00008000:00000010:3.0:1775768238.770364:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994209270a90. 00000080:00000010:3.0:1775768238.770369:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942090e33c0. 00000100:00080000:0.0:1775768238.770370:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775768238.770373:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000020:00000010:3.0:1775768238.770374:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcf000. 00000100:00080000:0.0:1775768238.770376:0:36:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000020:00000010:3.0:1775768238.770378:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff99420693e000. 00000100:00000001:0.0:1775768238.770380:0:36:0:(import.c:484:import_select_connection()) Process entered 00000080:00000001:3.0:1775768238.770382:0:13471:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1775768238.770384:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5491 00000080:00000001:3.0:1775768238.770387:0:13471:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768238.770392:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768238.770396:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768238.770399:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768238.770401:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768238.770404:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768238.770405:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768238.770407:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768238.770409:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768238.770411:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:0.0:1775768238.770413:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768238.770415:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:0.0:1775768238.770417:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:0.0:1775768238.770420:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:0.0:1775768238.770423:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768238.770425:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768238.770429:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768238.770431:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768238.770433:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:0.0:1775768238.770437:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768238.770439:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768238.770441:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768238.770444:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768238.770446:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768238.770448:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775768238.770450:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:0.0:1775768238.770452:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768238.770453:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768238.770457:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fc700. 00000020:00000040:0.0:1775768238.770458:0:36:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:0.0:1775768238.770460:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768238.770462:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768238.770463:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768238.770465:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768238.770466:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768238.770471:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220028400. 00000100:00000001:0.0:1775768238.770474:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768238.770479:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fc700 x1862022294291712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768238.770486:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768238.770489:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768238.770492:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003283b20a] to pc [ptlrpcd_rcv+-1] req@ffff9942099fc700 x1862022294291712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768238.770500:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768238.770506:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5501) 00000100:00000001:1.0:1775768238.770985:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768238.770991:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768238.770995:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768238.771003:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fc000 x1862022294291200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768238.771013:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768238.771015:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768238.771020:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768238.771023:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768238.771026:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768238.771030:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294291200:192.168.202.156@tcp:250:kworker.0 00000100:00000001:1.0:1775768238.771034:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768238.771040:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768238.771043:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768238.771047:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768238.771054:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543fc00. 02000000:00000001:1.0:1775768238.771056:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768238.771063:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768238.771068:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c738. 00000400:00000010:1.0:1775768238.771077:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf65d8. 00000100:00000200:1.0:1775768238.771087:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294291200, portal 25 00000100:00000001:1.0:1775768238.771091:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768238.771093:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641196544 : -112966068355072 : ffff9942099fc000) 00000100:00000040:1.0:1775768238.771098:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fc000 x1862022294291200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768263 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768238.771107:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768238.771111:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768238.771114:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000100:00000200:1.0:1775768238.771118:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294291200, offset 0 00000400:00000200:1.0:1775768238.771125:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768238.771135:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768238.771143:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768238.771150:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768238.771178:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768238.771183:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768238.771186:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768238.771190:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225448:225448:256:4294967295] 192.168.202.156@tcp LPNI seq info [225444:225444:8:4294967295] 00000400:00000200:1.0:1775768238.771195:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768238.771206:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768238.771213:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768238.771219:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541300. 00000800:00000200:1.0:1775768238.771227:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768238.771230:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775768238.771236:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768238.771239:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1775768238.771243:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768238.771246:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541300. 00000400:00000200:1.0:1775768238.771252:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768238.771257:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768238.771263:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000400:00000010:1.0:1775768238.771265:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000100:00000001:1.0:1775768238.771270:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768238.771273:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fc000 x1862022294291200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768263 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768238.771284:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768238.771288:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fc000 x1862022294291200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768263 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768238.771296:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768238.771297:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768238.771305:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768238.771309:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768238.771312:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4300164927 00000100:00000010:1.0:1775768238.771317:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba4e0. 00000100:00000001:1.0:1775768238.771326:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775768238.771334:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4300164927 00000100:00000010:1.0:1775768238.771337:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba240. 00000100:00000001:1.0:1775768238.771340:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775768238.771344:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768238.771346:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768238.771350:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768238.771354:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768238/real 1775768238] req@ffff9942099fc000 x1862022294291200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768238.771365:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768238.771367:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768238.771369:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c738. 00000400:00000200:1.0:1775768238.771373:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf65d8 00000400:00000010:1.0:1775768238.771374:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf65d8. 00000100:00000001:1.0:1775768238.771378:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768238.771382:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fc000 x1862022294291200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768238.771392:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fc000 x1862022294291200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768238.771399:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768238.771400:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768238.771403:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768238.771405:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768238.771408:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fc000 x1862022294291200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768238.771415:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768238.771419:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fc000 x1862022294291200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768263 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768238.771850:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768238.771853:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768238.771855:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768238.771856:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768238.771859:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768238.771863:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768238.771867:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768238.771874:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768238.771877:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768238.771884:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fc000 x1862022294291200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768263 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768238.771892:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294291200:192.168.202.156@tcp:250:kworker.0 00000100:00000001:1.0:1775768238.771898:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768238.771902:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ff800 x1862022294291328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768238.771909:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768238.771911:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768238.771914:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768238.771918:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768238.771920:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768238.771923:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294291328:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768238.771927:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768238.771931:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768238.771934:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768238.771937:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768238.771942:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543f800. 02000000:00000001:1.0:1775768238.771944:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768238.771949:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768238.771952:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c738. 00000400:00000010:1.0:1775768238.771957:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf65d8. 00000100:00000200:1.0:1775768238.771962:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294291328, portal 10 00000100:00000001:1.0:1775768238.771964:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768238.771965:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641210880 : -112966068340736 : ffff9942099ff800) 00000100:00000040:1.0:1775768238.771972:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ff800 x1862022294291328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768263 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768238.771978:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768238.771982:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768238.771985:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000100:00000200:1.0:1775768238.771988:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294291328, offset 0 00000400:00000200:1.0:1775768238.771993:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768238.772000:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768238.772006:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768238.772010:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768238.772013:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768238.772017:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768238.772019:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768238.772024:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225449:225449:256:4294967295] 192.168.202.156@tcp LPNI seq info [225445:225445:8:4294967295] 00000400:00000200:1.0:1775768238.772029:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768238.772036:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768238.772043:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768238.772047:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541300. 00000800:00000200:1.0:1775768238.772053:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768238.772057:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775768238.772061:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768238.772065:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1775768238.772068:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768238.772070:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541300. 00000400:00000200:1.0:1775768238.772074:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768238.772078:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768238.772081:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000400:00000010:1.0:1775768238.772083:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000100:00000001:1.0:1775768238.772086:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768238.772089:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ff800 x1862022294291328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768263 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768238.772099:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768238.772103:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ff800 x1862022294291328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768263 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768238.772109:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768238.772111:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768238.772116:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768238.772121:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:1.0:1775768238.772124:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768238.772127:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300163604 00000100:00000001:1.0:1775768238.772130:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300164927 00000100:00000001:1.0:1775768238.772133:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300163604 00000100:00000001:1.0:1775768238.772135:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300164927 00000100:00000001:1.0:1775768238.772138:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768238.772140:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768238.772143:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768238.772147:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768238/real 1775768238] req@ffff9942099ff800 x1862022294291328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768238.772644:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768238.772646:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768238.772648:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c738. 00000400:00000200:1.0:1775768238.772653:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf65d8 00000400:00000010:1.0:1775768238.772654:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf65d8. 00000100:00000001:1.0:1775768238.772657:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768238.772662:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ff800 x1862022294291328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768238.772676:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ff800 x1862022294291328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768238.772682:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768238.772683:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768238.772685:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768238.772687:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768238.772690:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ff800 x1862022294291328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768238.772697:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768238.772701:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ff800 x1862022294291328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768263 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768238.772709:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768238.772711:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768238.772713:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768238.772714:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768238.772716:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768238.772721:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768238.772724:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768238.772727:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768238.772729:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768238.772733:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ff800 x1862022294291328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768263 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768238.772738:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294291328:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768238.772741:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768238.772745:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fd500 x1862022294291456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768238.772748:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768238.772749:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768238.772751:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768238.772754:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768238.772755:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768238.772757:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294291456:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768238.772759:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768238.772761:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768238.772763:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768238.772764:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768238.772767:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543d800. 02000000:00000001:1.0:1775768238.772769:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768238.772772:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768238.772773:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c738. 00000400:00000010:1.0:1775768238.772777:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf65d8. 00000100:00000200:1.0:1775768238.772779:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294291456, portal 10 00000100:00000001:1.0:1775768238.772781:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768238.772782:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641201920 : -112966068349696 : ffff9942099fd500) 00000100:00000040:1.0:1775768238.772786:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fd500 x1862022294291456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768263 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768238.772790:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768238.772792:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768238.772793:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000100:00000200:1.0:1775768238.772795:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294291456, offset 0 00000400:00000200:1.0:1775768238.772798:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768238.772801:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768238.772805:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768238.772807:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768238.772809:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768238.772811:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768238.772813:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768238.772815:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225450:225450:256:4294967295] 192.168.202.156@tcp LPNI seq info [225446:225446:8:4294967295] 00000400:00000200:1.0:1775768238.772819:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768238.772838:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768238.772844:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768238.772848:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541300. 00000800:00000200:1.0:1775768238.772853:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768238.772856:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775768238.772860:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768238.772863:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1775768238.772867:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768238.772869:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541300. 00000400:00000200:1.0:1775768238.772872:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768238.772875:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768238.772878:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000400:00000010:1.0:1775768238.772879:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000100:00000001:1.0:1775768238.772880:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768238.772883:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fd500 x1862022294291456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768263 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768238.772888:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768238.772890:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fd500 x1862022294291456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768263 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768238.772894:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768238.772895:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768238.772897:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768238.772900:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:1.0:1775768238.772902:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768238.772904:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300163597 00000100:00000001:1.0:1775768238.772906:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300164927 00000100:00000001:1.0:1775768238.772916:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300163597 00000100:00000001:1.0:1775768238.772917:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300164927 00000100:00000001:1.0:1775768238.772919:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768238.772920:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768238.772922:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768238.772924:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768238/real 1775768238] req@ffff9942099fd500 x1862022294291456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768238.772929:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768238.772930:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768238.772931:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c738. 00000400:00000200:1.0:1775768238.772933:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf65d8 00000400:00000010:1.0:1775768238.772934:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf65d8. 00000100:00000001:1.0:1775768238.772936:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768238.772938:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fd500 x1862022294291456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768238.772943:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fd500 x1862022294291456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768238.772946:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768238.772947:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768238.772948:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768238.772949:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768238.772950:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fd500 x1862022294291456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768238.772954:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768238.772956:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fd500 x1862022294291456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768263 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768238.772959:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768238.772961:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768238.772962:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768238.772963:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768238.772964:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768238.772968:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768238.772969:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768238.772971:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768238.772972:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768238.772974:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fd500 x1862022294291456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768263 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768238.772979:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294291456:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768238.772981:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768238.772983:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ff100 x1862022294291584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768238.772987:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768238.772988:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768238.772989:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768238.772990:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768238.772991:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768238.772993:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294291584:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768238.772995:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768238.772996:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768238.772997:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768238.772998:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768238.773001:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543c400. 02000000:00000001:1.0:1775768238.773002:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768238.773004:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768238.773006:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c738. 00000400:00000010:1.0:1775768238.773008:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf65d8. 00000100:00000200:1.0:1775768238.773010:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294291584, portal 4 00000100:00000001:1.0:1775768238.773012:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768238.773013:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641209088 : -112966068342528 : ffff9942099ff100) 00000100:00000040:1.0:1775768238.773016:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ff100 x1862022294291584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768263 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768238.773021:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768238.773022:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768238.773023:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000100:00000200:1.0:1775768238.773025:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294291584, offset 0 00000400:00000200:1.0:1775768238.773027:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768238.773029:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768238.773031:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768238.773033:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768238.773035:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768238.773037:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768238.773039:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768238.773042:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225451:225451:256:4294967295] 192.168.202.156@tcp LPNI seq info [225447:225447:8:4294967295] 00000400:00000200:1.0:1775768238.773046:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768238.773052:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768238.773056:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768238.773059:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541300. 00000800:00000200:1.0:1775768238.773063:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768238.773065:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775768238.773069:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768238.773071:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1775768238.773075:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768238.773077:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541300. 00000400:00000200:1.0:1775768238.773079:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768238.773083:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768238.773085:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000400:00000010:1.0:1775768238.773087:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000100:00000001:1.0:1775768238.773089:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768238.773093:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ff100 x1862022294291584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768263 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768238.773104:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768238.773106:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ff100 x1862022294291584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768263 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768238.773118:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768238.773119:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768238.773124:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768238.773126:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:1.0:1775768238.773129:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768238.773130:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300163620 00000100:00000001:1.0:1775768238.773132:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300164927 00000100:00000001:1.0:1775768238.773133:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300163620 00000100:00000001:1.0:1775768238.773135:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300164927 00000100:00000001:1.0:1775768238.773136:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768238.773137:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768238.773138:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768238.773140:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768238/real 1775768238] req@ffff9942099ff100 x1862022294291584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768238.773145:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768238.773146:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768238.773147:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c738. 00000400:00000200:1.0:1775768238.773149:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf65d8 00000400:00000010:1.0:1775768238.773150:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf65d8. 00000100:00000001:1.0:1775768238.773151:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768238.773153:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ff100 x1862022294291584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768238.773191:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ff100 x1862022294291584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768238.773198:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768238.773200:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768238.773202:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768238.773204:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768238.773207:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ff100 x1862022294291584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768238.773213:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768238.773216:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ff100 x1862022294291584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768263 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768238.773223:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768238.773225:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768238.773227:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768238.773228:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768238.773230:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768238.773234:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768238.773235:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768238.773238:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768238.773240:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768238.773242:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ff100 x1862022294291584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768263 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768238.773247:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294291584:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768238.773250:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768238.773253:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fc700 x1862022294291712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768238.773257:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768238.773258:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768238.773260:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768238.773261:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768238.773263:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768238.773265:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294291712:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768238.773267:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768238.773269:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768238.773271:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768238.773272:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768238.773276:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543c800. 02000000:00000001:1.0:1775768238.773277:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768238.773279:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768238.773281:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c738. 00000400:00000010:1.0:1775768238.773283:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf65d8. 00000100:00000200:1.0:1775768238.773286:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294291712, portal 4 00000100:00000001:1.0:1775768238.773287:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768238.773288:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641198336 : -112966068353280 : ffff9942099fc700) 00000100:00000040:1.0:1775768238.773292:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fc700 x1862022294291712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768263 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768238.773296:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768238.773297:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768238.773299:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000100:00000200:1.0:1775768238.773300:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294291712, offset 0 00000400:00000200:1.0:1775768238.773302:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768238.773305:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768238.773307:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768238.773308:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768238.773310:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768238.773312:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768238.773314:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768238.773317:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225452:225452:256:4294967295] 192.168.202.156@tcp LPNI seq info [225448:225448:8:4294967295] 00000400:00000200:1.0:1775768238.773321:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768238.773327:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768238.773331:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768238.773334:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541300. 00000800:00000200:1.0:1775768238.773338:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768238.773340:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775768238.773344:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768238.773347:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1775768238.773350:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768238.773352:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541300. 00000400:00000200:1.0:1775768238.773355:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768238.773358:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768238.773361:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000400:00000010:1.0:1775768238.773363:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000100:00000001:1.0:1775768238.773365:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768238.773375:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fc700 x1862022294291712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768263 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768238.773383:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768238.773385:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fc700 x1862022294291712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768263 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768238.773389:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768238.773390:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768238.773392:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768238.773394:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:1.0:1775768238.773397:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768238.773399:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300163624 00000100:00000001:1.0:1775768238.773401:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300164927 00000100:00000001:1.0:1775768238.773402:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300163624 00000100:00000001:1.0:1775768238.773404:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300164927 00000100:00000001:1.0:1775768238.773405:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768238.773407:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768238.773408:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768238.773410:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768238/real 1775768238] req@ffff9942099fc700 x1862022294291712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768238.773414:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768238.773415:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768238.773416:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c738. 00000400:00000200:1.0:1775768238.773418:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf65d8 00000400:00000010:1.0:1775768238.773419:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf65d8. 00000100:00000001:1.0:1775768238.773420:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768238.773422:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fc700 x1862022294291712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768238.773426:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fc700 x1862022294291712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768238.773430:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768238.773431:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768238.773432:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768238.773433:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768238.773434:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fc700 x1862022294291712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768238.773438:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768238.773440:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fc700 x1862022294291712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768263 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768238.773443:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768238.773444:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768238.773445:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768238.773446:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768238.773447:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768238.773450:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768238.773451:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768238.773453:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768238.773454:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768238.773456:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fc700 x1862022294291712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768263 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768238.773464:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294291712:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768238.773467:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768238.773469:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768238.773471:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fc000 x1862022294291200/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768263 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768238.773475:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768238.773477:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768238.773479:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543fc00. 02000000:00000001:1.0:1775768238.773481:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768238.773483:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768238.773484:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:1.0:1775768238.773485:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768238.773487:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002bc00. 02000000:00000001:1.0:1775768238.773490:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768238.773492:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768238.773493:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fc000. 00000100:00000001:1.0:1775768238.773496:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768238.773497:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768238.773498:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768238.773500:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ff800 x1862022294291328/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768263 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768238.773504:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768238.773505:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768238.773506:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543f800. 02000000:00000001:1.0:1775768238.773507:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768238.773508:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768238.773508:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:1.0:1775768238.773509:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768238.773510:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029800. 02000000:00000001:1.0:1775768238.773512:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768238.773512:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768238.773513:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ff800. 00000100:00000001:1.0:1775768238.773515:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768238.773516:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768238.773517:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768238.773518:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fd500 x1862022294291456/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768263 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768238.773523:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768238.773524:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768238.773525:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543d800. 02000000:00000001:1.0:1775768238.773526:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768238.773528:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768238.773529:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:1.0:1775768238.773531:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768238.773532:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029400. 02000000:00000001:1.0:1775768238.773534:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768238.773535:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768238.773536:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fd500. 00000100:00000001:1.0:1775768238.773538:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768238.773539:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768238.773541:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768238.773543:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ff100 x1862022294291584/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768263 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768238.773549:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768238.773550:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768238.773551:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543c400. 02000000:00000001:1.0:1775768238.773553:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768238.773554:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768238.773555:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:1.0:1775768238.773557:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768238.773558:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002a800. 02000000:00000001:1.0:1775768238.773560:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768238.773561:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768238.773562:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ff100. 00000100:00000001:1.0:1775768238.773565:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768238.773566:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768238.773567:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768238.773570:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fc700 x1862022294291712/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768263 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768238.773577:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768238.773578:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768238.773579:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543c800. 02000000:00000001:1.0:1775768238.773581:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768238.773582:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768238.773582:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:1.0:1775768238.773595:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768238.773597:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028400. 02000000:00000001:1.0:1775768238.773600:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768238.773601:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768238.773601:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fc700. 00000100:00000001:1.0:1775768238.773603:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768238.773603:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768238.773605:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768238.773612:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768238.773613:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768238.773615:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768238.773616:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768238.833305:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768238.833313:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768238.833321:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768238.833323:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768238.833327:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768238.833328:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768238.833332:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768238.833333:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768238.833336:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768238.833338:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768238.833393:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768238.833396:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768238.833398:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768238.897289:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768238.897298:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768238.897300:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768238.897308:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768238.897312:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768238.897318:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768238.897321:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768238.897323:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768238.897326:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768239.793245:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768239.793253:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768239.793265:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768239.793271:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768239.793276:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768239.793280:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768239.793282:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768239.793284:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768239.793288:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768239.793310:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768239.793318:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768239.793321:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768239.793327:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768239.793331:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768239.793337:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768239.793338:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768239.793342:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768239.793344:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768239.793345:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768239.793350:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768239.793351:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775768239.793360:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768239.793362:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768239.793363:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768239.793373:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768239.793375:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768239.793377:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768239.857256:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768239.857262:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768239.857267:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768239.857269:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768239.857273:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768239.857274:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768239.857277:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768239.857278:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768239.857281:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768239.857282:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768239.857339:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768239.857342:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768239.857345:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768239.921246:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768239.921251:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768239.921252:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768239.921256:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768239.921259:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768239.921262:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768239.921263:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768239.921265:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768239.921267:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768240.113248:0:15036:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4300164927, at 4300164928 00000100:00000040:1.0:1775768240.113248:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4300164927, at 4300164928 00000100:00000010:1.0:1775768240.113261:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cba240. 00000100:00000010:0.0:1775768240.113263:0:15036:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cba4e0. 00000100:00000001:3.0:1775768240.817280:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768240.817281:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768240.817287:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768240.817289:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768240.817295:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:1.0:1775768240.817297:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775768240.817298:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768240.817304:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1775768240.817305:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768240.817309:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768240.817311:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768240.817311:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768240.817312:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768240.817319:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768240.817323:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768240.817326:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768240.817330:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768240.817332:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768240.817334:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768240.817335:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768240.817343:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768240.817344:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775768240.817350:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768240.817364:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768240.817367:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768240.817369:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768240.817372:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768240.881296:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768240.881302:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768240.881308:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768240.881310:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768240.881315:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768240.881317:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768240.881320:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768240.881321:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768240.881325:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768240.881326:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768240.881380:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768240.881385:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768240.881388:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768240.945309:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768240.945314:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768240.945315:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768240.945319:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768240.945321:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768240.945325:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768240.945327:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768240.945328:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768240.945330:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768241.841290:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768241.841291:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768241.841293:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768241.841301:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768241.841304:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768241.841304:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768241.841305:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768241.841308:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768241.841309:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768241.841313:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:1.0:1775768241.841315:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:2.0:1775768241.841320:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768241.841322:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768241.841322:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768241.841331:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768241.841331:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768241.841336:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768241.841338:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768241.841339:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768241.841344:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768241.841345:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768241.841345:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768241.841350:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768241.841355:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768241.841358:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768241.841361:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768241.841365:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768241.905300:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768241.905309:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768241.905312:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768241.905315:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768241.905321:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768241.905323:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768241.905326:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768241.905329:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768241.905342:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768241.905343:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768241.905348:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768241.905349:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768241.905352:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768241.905354:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768241.905357:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768241.905359:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768241.905421:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768241.905425:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768241.905428:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768241.969307:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768241.969317:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768241.969319:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768241.969326:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768241.969332:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768241.969337:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768241.969341:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768241.969344:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768241.969347:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768242.865235:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768242.865240:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768242.865242:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768242.865248:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775768242.865251:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768242.865256:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768242.865258:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768242.865260:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768242.865262:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768242.865279:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768242.865280:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768242.865288:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768242.865288:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768242.865291:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768242.865292:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768242.865299:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:0.0:1775768242.865300:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768242.865306:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768242.865307:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768242.865316:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768242.865316:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768242.865321:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768242.865324:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768242.865327:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768242.865330:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768242.865332:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768242.865335:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768242.929277:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768242.929283:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768242.929295:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768242.929299:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768242.929302:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768242.929302:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768242.929305:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768242.929306:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768242.929308:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768242.929309:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768242.929381:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768242.929385:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768242.929387:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768242.993237:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768242.993244:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768242.993245:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768242.993252:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768242.993256:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768242.993259:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768242.993261:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768242.993263:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768242.993266:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768243.889253:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768243.889259:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768243.889263:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768243.889269:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:2.0:1775768243.889269:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000080:00000001:3.0:1775768243.889272:0:13471:0:(rw.c:1481:ll_writepages()) Process entered 00000100:00000001:1.0:1775768243.889274:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768243.889275:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000080:00000001:3.0:1775768243.889276:0:13471:0:(file.c:5250:cl_sync_file_range()) Process entered 00000100:00000001:1.0:1775768243.889277:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768243.889280:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768243.889281:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000020:00000001:3.0:1775768243.889281:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00000001:1.0:1775768243.889286:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768243.889286:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768243.889290:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768243.889290:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:3.0:1775768243.889290:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff99420693e000. 00000100:00000001:0.0:1775768243.889295:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768243.889297:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768243.889297:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1775768243.889297:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dce800. 00000100:00000001:1.0:1775768243.889300:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:3.0:1775768243.889304:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210dcf800. 00000100:00000001:1.0:1775768243.889306:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768243.889307:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000020:00000010:3.0:1775768243.889308:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207981e40. 00000100:00000001:0.0:1775768243.889313:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:3.0:1775768243.889316:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994208b50000. 00000100:00100000:0.0:1775768243.889319:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00020000:00000010:3.0:1775768243.889321:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994218000300. 00000100:00080000:2.0:1775768243.889322:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775768243.889329:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00008000:00000010:3.0:1775768243.889330:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994207e08e20. 00000100:00080000:2.0:1775768243.889331:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775768243.889336:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768243.889337:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000080:00000010:3.0:1775768243.889338:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99420541b1f8. 00000100:00080000:2.0:1775768243.889342:0:36:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768243.889345:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000080:00000010:3.0:1775768243.889346:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077c0248. 00000100:00000001:0.0:1775768243.889348:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768243.889348:0:36:0:(import.c:484:import_select_connection()) Process entered 00000020:00000010:3.0:1775768243.889350:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcea00. 00000100:00000001:0.0:1775768243.889355:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:3.0:1775768243.889356:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942086f8420. 00000100:00080000:2.0:1775768243.889361:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5496 00020000:00000010:3.0:1775768243.889362:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994207e8e400. 00008000:00000010:3.0:1775768243.889366:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994209270a90. 00000080:00000010:3.0:1775768243.889370:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942090e33c0. 00000400:00000200:2.0:1775768243.889374:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:3.0:1775768243.889374:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107590086664 : -112966119464952 : ffff99420693e008) 00000020:00001000:3.0:1775768243.889378:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff99420693e000 00000020:00000001:3.0:1775768243.889382:0:13471:0:(cl_object.c:151:cl_object_top()) ffff9942078a76c0 -> ffff9942078a76c0 00000400:00000200:2.0:1775768243.889384:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000020:00000001:3.0:1775768243.889385:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:3.0:1775768243.889387:0:13471:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:3.0:1775768243.889388:0:13471:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:3.0:1775768243.889390:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:3.0:1775768243.889391:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:3.0:1775768243.889392:0:13471:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:3.0:1775768243.889395:0:13471:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017a:0x0]io ffff9942077c02a8 type 6 ignore/verify layout 1/0 00020000:00000002:3.0:1775768243.889397:0:13471:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207c1f9b0) owner 0000000000000000/ffff994220080000 00020000:00000001:3.0:1775768243.889399:0:13471:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:3.0:1775768243.889402:0:13471:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00000100:00080000:2.0:1775768243.889404:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00020000:00000001:3.0:1775768243.889404:0:13471:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00020000:00000001:3.0:1775768243.889406:0:13471:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768243.889408:0:13471:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1775768243.889410:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00020000:00000001:3.0:1775768243.889411:0:13471:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:3.0:1775768243.889413:0:13471:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768243.889415:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:3.0:1775768243.889416:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:3.0:1775768243.889417:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:3.0:1775768243.889419:0:13471:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00000100:00000040:2.0:1775768243.889421:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00020000:00000002:3.0:1775768243.889421:0:13471:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207c1f9b0) owner 0000000000000000/ffff994220080000 00000100:00000001:2.0:1775768243.889426:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775768243.889427:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768243.889430:0:13471:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:2.0:1775768243.889432:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000001:3.0:1775768243.889432:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000020:00000040:2.0:1775768243.889435:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00020000:00000001:3.0:1775768243.889435:0:13471:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00020000:00200000:3.0:1775768243.889438:0:13471:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000020:00000001:2.0:1775768243.889439:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00020000:00000002:3.0:1775768243.889441:0:13471:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00000020:00000040:2.0:1775768243.889442:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00020000:00000001:3.0:1775768243.889445:0:13471:0:(lov_io.c:155:lov_sub_get()) Process entered 00000020:00000001:2.0:1775768243.889446:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00020000:00000001:3.0:1775768243.889447:0:13471:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000020:00000001:3.0:1775768243.889448:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000001:2.0:1775768243.889451:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000010:3.0:1775768243.889451:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff99420693ecf0. 00000100:00000040:2.0:1775768243.889456:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768243.889458:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768243.889460:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768243.889461:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000010:3.0:1775768243.889464:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcf200. 00000020:00000040:2.0:1775768243.889465:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:2.0:1775768243.889469:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000020:00000010:3.0:1775768243.889473:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210dce000. 00000020:00000010:3.0:1775768243.889476:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207981900. 00000100:00000001:2.0:1775768243.889478:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:3.0:1775768243.889480:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994208b51650. 02000000:00000001:2.0:1775768243.889482:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00020000:00000010:3.0:1775768243.889484:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994218000000. 02000000:00000001:2.0:1775768243.889486:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00008000:00000010:3.0:1775768243.889488:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994207e0ade8. 02000000:00000001:2.0:1775768243.889490:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:3.0:1775768243.889493:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99420541b0a8. 00000100:00000001:2.0:1775768243.889494:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775768243.889497:0:36:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:3.0:1775768243.889498:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077c1da8. 00000020:00000010:3.0:1775768243.889503:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcf400. 00000100:00000010:2.0:1775768243.889504:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dff480. 00000008:00000010:3.0:1775768243.889507:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942086f8160. 00000020:00000040:2.0:1775768243.889508:0:36:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 00020000:00000010:3.0:1775768243.889513:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994207e8ed00. 02000000:00000001:2.0:1775768243.889514:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775768243.889517:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:3.0:1775768243.889517:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994209270ef0. 00000100:00000001:2.0:1775768243.889520:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000080:00000010:3.0:1775768243.889522:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942090e3960. 02000000:00000001:2.0:1775768243.889524:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000020:00000001:3.0:1775768243.889526:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107590089976 : -112966119461640 : ffff99420693ecf8) 02000000:00000001:2.0:1775768243.889529:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:3.0:1775768243.889529:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff99420693ecf0 00000020:00000001:3.0:1775768243.889531:0:13471:0:(cl_object.c:151:cl_object_top()) ffff9942078a7788 -> ffff9942078a76c0 00000020:00000001:3.0:1775768243.889546:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 02000000:00000010:2.0:1775768243.889549:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1f400. 00000020:00000001:3.0:1775768243.889549:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:3.0:1775768243.889551:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:3.0:1775768243.889553:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768243.889555:0:13471:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768243.889556:0:13471:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107612435560 : -112966097116056 : ffff994207e8e468) 00000100:00000001:2.0:1775768243.889557:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768243.889559:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000020:00000001:3.0:1775768243.889561:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768243.889562:0:13471:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768243.889563:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768243.889565:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00020000:00000001:3.0:1775768243.889566:0:13471:0:(lov_io.c:1108:lov_io_lock()) Process entered 00000100:00100000:2.0:1775768243.889568:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dff480 x1862022294291840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00020000:00000001:3.0:1775768243.889568:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775768243.889570:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:3.0:1775768243.889572:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:3.0:1775768243.889574:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768243.889577:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768243.889579:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768243.889580:0:13471:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768243.889581:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:3.0:1775768243.889582:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768243.889583:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768243.889584:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00020000:00000001:3.0:1775768243.889586:0:13471:0:(lov_io.c:1114:lov_io_start()) Process entered 00020000:00000001:3.0:1775768243.889588:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775768243.889589:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00000100:00000001:2.0:1775768243.889591:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000008:00000001:3.0:1775768243.889591:0:13471:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000008:00000001:3.0:1775768243.889593:0:13471:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000100:00000001:2.0:1775768243.889597:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1775768243.889598:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:3.0:1775768243.889601:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:3.0:1775768243.889602:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768243.889605:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093a15639] to pc [ptlrpcd_rcv+-1] req@ffff994207dff480 x1862022294291840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1775768243.889605:0:13471:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000008:00000001:3.0:1775768243.889607:0:13471:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000008:00000001:3.0:1775768243.889608:0:13471:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107884691392 : -112965824860224 : ffff994218232fc0) 00000008:00000020:3.0:1775768243.889612:0:13471:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994218232fc0 ready 0|-|- wr 1|-|+ rd 0|- 10 in flight 00000008:00000001:3.0:1775768243.889618:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:3.0:1775768243.889619:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:3.0:1775768243.889621:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1775768243.889624:0:13471:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000008:00000020:3.0:1775768243.889626:0:13471:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994210675880 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000008:00000001:3.0:1775768243.889629:0:13471:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1775768243.889630:0:13471:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768243.889631:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768243.889632:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768243.889633:0:13471:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768243.889634:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768243.889635:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00020000:00000001:3.0:1775768243.889636:0:13471:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00020000:00000001:3.0:1775768243.889637:0:13471:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000020:00000001:3.0:1775768243.889637:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000001:3.0:1775768243.889638:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00020000:00000001:3.0:1775768243.889639:0:13471:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768243.889639:0:13471:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000020:00000001:3.0:1775768243.889640:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00000020:00000001:3.0:1775768243.889641:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00020000:00000001:3.0:1775768243.889643:0:13471:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00020000:00000001:3.0:1775768243.889643:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775768243.889644:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000100:00000001:2.0:1775768243.889645:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768243.889645:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:3.0:1775768243.889647:0:13471:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768243.889649:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768243.889650:0:13471:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000020:00000001:3.0:1775768243.889650:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:3.0:1775768243.889652:0:13471:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000020:00000001:3.0:1775768243.889653:0:13471:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000100:00080000:2.0:1775768243.889654:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000001:3.0:1775768243.889654:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:3.0:1775768243.889657:0:13471:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00000100:00080000:2.0:1775768243.889660:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00020000:00000001:3.0:1775768243.889660:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000001:2.0:1775768243.889663:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000020:00000001:3.0:1775768243.889663:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000001:3.0:1775768243.889665:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000100:00080000:2.0:1775768243.889666:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768243.889667:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00020000:00000001:3.0:1775768243.889668:0:13471:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768243.889672:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768243.889672:0:36:0:(import.c:484:import_select_connection()) Process entered 00020000:00000001:3.0:1775768243.889672:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768243.889676:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00020000:00000001:3.0:1775768243.889676:0:13471:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000100:00080000:2.0:1775768243.889678:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5496 00000020:00000001:3.0:1775768243.889679:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000020:00000001:3.0:1775768243.889682:0:13471:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768243.889683:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dff480 x1862022294291840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775768243.889685:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:3.0:1775768243.889689:0:13471:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:3.0:1775768243.889692:0:13471:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000400:00000200:2.0:1775768243.889693:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:3.0:1775768243.889695:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00000100:00000001:1.0:1775768243.889698:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000001:3.0:1775768243.889698:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000100:00000001:1.0:1775768243.889701:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768243.889702:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000020:00001000:3.0:1775768243.889702:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff99420693ecf0 00000100:00080000:2.0:1775768243.889707:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000020:00000010:3.0:1775768243.889708:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210dce000. 02000000:00000001:1.0:1775768243.889709:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768243.889713:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1775768243.889713:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768243.889717:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000010:3.0:1775768243.889717:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207981900. 02000000:00000001:1.0:1775768243.889718:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768243.889721:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000008:00000010:3.0:1775768243.889722:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994208b51650. 00000100:00100000:1.0:1775768243.889726:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294291840:192.168.202.156@tcp:250:kworker.0 00000100:00000001:2.0:1775768243.889727:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00020000:00000010:3.0:1775768243.889728:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994218000000. 00000020:00000001:2.0:1775768243.889732:0:36:0:(genops.c:811:class_conn2export()) Process entered 00008000:00000010:3.0:1775768243.889733:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994207e0ade8. 00000020:00000040:2.0:1775768243.889734:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000100:00000001:1.0:1775768243.889735:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000080:00000010:3.0:1775768243.889736:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99420541b0a8. 00000020:00000001:2.0:1775768243.889738:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000080:00000010:3.0:1775768243.889739:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077c1da8. 02000000:00000001:1.0:1775768243.889740:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:2.0:1775768243.889741:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000010:3.0:1775768243.889743:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcf200. 02000000:00000001:1.0:1775768243.889744:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768243.889745:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 02000000:00000001:1.0:1775768243.889748:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000010:3.0:1775768243.889748:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942086f8160. 00000020:00000001:2.0:1775768243.889749:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00020000:00000010:3.0:1775768243.889752:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994207e8ed00. 02000000:00000010:1.0:1775768243.889753:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543c400. 00000100:00000040:2.0:1775768243.889755:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 02000000:00000001:1.0:1775768243.889756:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:3.0:1775768243.889756:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994209270ef0. 00000100:00000001:2.0:1775768243.889761:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000080:00000010:3.0:1775768243.889761:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942090e3960. 00000400:00000200:1.0:1775768243.889765:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000010:3.0:1775768243.889765:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcf400. 00000400:00000010:1.0:1775768243.889769:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c738. 00000020:00000010:3.0:1775768243.889769:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff99420693ecf0. 00000100:00000040:2.0:1775768243.889773:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00020000:00000001:3.0:1775768243.889774:0:13471:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00000400:00000010:1.0:1775768243.889776:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf65d8. 00000100:00000001:2.0:1775768243.889778:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00020000:00000001:3.0:1775768243.889778:0:13471:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000020:00000040:2.0:1775768243.889782:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000080:00000001:3.0:1775768243.889782:0:13471:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000100:00000200:1.0:1775768243.889783:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294291840, portal 25 00000080:00200000:3.0:1775768243.889785:0:13471:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00000001:1.0:1775768243.889788:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768243.889790:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611849856 : -112966097701760 : ffff994207dff480) 00000100:00080000:2.0:1775768243.889790:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000080:00000001:3.0:1775768243.889792:0:13471:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:3.0:1775768243.889795:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000100:00000040:1.0:1775768243.889799:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dff480 x1862022294291840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768268 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768243.889799:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:3.0:1775768243.889800:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff99420693e000 02000000:00000001:2.0:1775768243.889804:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000020:00000010:3.0:1775768243.889805:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210dcf800. 00000020:00000010:3.0:1775768243.889810:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207981e40. 02000000:00000001:2.0:1775768243.889811:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768243.889815:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:2.0:1775768243.889815:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:3.0:1775768243.889816:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994208b50000. 00000100:00000001:2.0:1775768243.889819:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00020000:00000010:3.0:1775768243.889820:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994218000300. 00000100:00000040:1.0:1775768243.889823:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000008:00100000:2.0:1775768243.889824:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00008000:00000010:3.0:1775768243.889824:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994207e08e20. 00000400:00000010:1.0:1775768243.889828:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000008:00000001:2.0:1775768243.889829:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:3.0:1775768243.889829:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99420541b1f8. 00000080:00000010:3.0:1775768243.889834:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077c0248. 00000100:00000200:1.0:1775768243.889835:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294291840, offset 0 00000100:00000001:2.0:1775768243.889835:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1775768243.889837:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dce800. 00000008:00000010:3.0:1775768243.889841:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942086f8420. 00000100:00000010:2.0:1775768243.889842:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfce00. 00020000:00000010:3.0:1775768243.889843:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994207e8e400. 00000400:00000200:1.0:1775768243.889847:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000020:00000040:2.0:1775768243.889847:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00008000:00000010:3.0:1775768243.889848:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994209270a90. 00000080:00000010:3.0:1775768243.889852:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942090e33c0. 00000020:00000010:3.0:1775768243.889855:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcea00. 00000020:00000010:3.0:1775768243.889856:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff99420693e000. 00000080:00000001:3.0:1775768243.889858:0:13471:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1775768243.889859:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:2.0:1775768243.889860:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000080:00000001:3.0:1775768243.889862:0:13471:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768243.889864:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768243.889867:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:1.0:1775768243.889868:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 02000000:00000001:2.0:1775768243.889870:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000080:00000001:3.0:1775768243.889870:0:13471:0:(rw.c:1481:ll_writepages()) Process entered 00000080:00000001:3.0:1775768243.889873:0:13471:0:(file.c:5250:cl_sync_file_range()) Process entered 00000400:00000200:1.0:1775768243.889874:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:2.0:1775768243.889874:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768243.889875:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 02000000:00000010:2.0:1775768243.889881:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1f000. 00000020:00000010:3.0:1775768243.889881:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff99420693e000. 00000400:00000200:1.0:1775768243.889882:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000100:00000001:2.0:1775768243.889888:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768243.889889:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768243.889895:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00100000:2.0:1775768243.889896:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfce00 x1862022294291968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000010:3.0:1775768243.889897:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcea00. 00000400:00000200:1.0:1775768243.889902:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225453:225453:256:4294967295] 192.168.202.156@tcp LPNI seq info [225449:225449:8:4294967295] 00000020:00000010:3.0:1775768243.889905:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210dce800. 00000020:00000010:3.0:1775768243.889910:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207981e40. 00000008:00000010:3.0:1775768243.889915:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994208b50000. 00000400:00000200:1.0:1775768243.889916:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768243.889917:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00020000:00000010:3.0:1775768243.889919:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994218000300. 00000100:00000001:2.0:1775768243.889921:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:3.0:1775768243.889922:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994207e08e20. 00000080:00000010:3.0:1775768243.889927:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99420541b1f8. 00000100:00000040:2.0:1775768243.889928:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e5a7debe] to pc [ptlrpcd_rcv+-1] req@ffff994207dfce00 x1862022294291968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000010:3.0:1775768243.889930:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077c0248. 00000400:00000200:1.0:1775768243.889934:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000020:00000010:3.0:1775768243.889935:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcf800. 00000008:00000010:3.0:1775768243.889939:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942086f8420. 00020000:00000010:3.0:1775768243.889943:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994207e8e400. 00008000:00000010:3.0:1775768243.889947:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994209270a90. 00000800:00000200:1.0:1775768243.889949:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000080:00000010:3.0:1775768243.889951:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942090e33c0. 00000100:00000001:2.0:1775768243.889953:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768243.889953:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107590086664 : -112966119464952 : ffff99420693e008) 00000800:00000010:1.0:1775768243.889954:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541300. 00000100:00080000:2.0:1775768243.889959:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00001000:3.0:1775768243.889959:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff99420693e000 00000020:00000001:3.0:1775768243.889964:0:13471:0:(cl_object.c:151:cl_object_top()) ffff9942078a7f58 -> ffff9942078a7f58 00000800:00000200:1.0:1775768243.889965:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000001:3.0:1775768243.889967:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000100:00080000:2.0:1775768243.889968:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000200:1.0:1775768243.889969:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000080:00000001:3.0:1775768243.889970:0:13471:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000100:00000001:2.0:1775768243.889971:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000080:00200000:3.0:1775768243.889972:0:13471:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017b:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000100:00080000:2.0:1775768243.889974:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000020:00000001:3.0:1775768243.889976:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000800:00000200:1.1:1775768243.889977:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000001:3.0:1775768243.889979:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00000001:2.0:1775768243.889980:0:36:0:(import.c:484:import_select_connection()) Process entered 00000080:00000001:3.0:1775768243.889981:0:13471:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:3.0:1775768243.889984:0:13471:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017b:0x0]io ffff9942077c02a8 type 6 ignore/verify layout 1/0 00020000:00000002:3.0:1775768243.889987:0:13471:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207c1f000) owner 0000000000000000/ffff994220080000 00020000:00000001:3.0:1775768243.889988:0:13471:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:3.0:1775768243.889991:0:13471:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00000800:00000200:1.1:1775768243.889991:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000100:00080000:2.0:1775768243.889992:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5496 00020000:00000001:3.0:1775768243.889992:0:13471:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00020000:00000001:3.0:1775768243.889996:0:13471:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1775768243.890000:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00020000:00000001:3.0:1775768243.890000:0:13471:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1775768243.890001:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000800:00000010:1.0:1775768243.890004:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541300. 00020000:00000001:3.0:1775768243.890006:0:13471:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00000400:00000200:2.0:1775768243.890008:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000400:00000200:1.0:1775768243.890009:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00020000:00000001:3.0:1775768243.890009:0:13471:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768243.890014:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000020:00000001:3.0:1775768243.890014:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000400:00000200:1.0:1775768243.890017:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000020:00000001:3.0:1775768243.890018:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:3.0:1775768243.890020:0:13471:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00000100:00000040:2.0:1775768243.890021:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000200:1.0:1775768243.890022:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00020000:00000002:3.0:1775768243.890023:0:13471:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207c1f000) owner 0000000000000000/ffff994220080000 00000400:00000010:1.0:1775768243.890026:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000100:00000001:2.0:1775768243.890026:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:3.0:1775768243.890028:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768243.890031:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:2.0:1775768243.890032:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000001:3.0:1775768243.890032:0:13471:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:3.0:1775768243.890035:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000100:00000200:1.0:1775768243.890037:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dff480 x1862022294291840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768268 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768243.890037:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00020000:00000001:3.0:1775768243.890038:0:13471:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00020000:00200000:3.0:1775768243.890041:0:13471:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000020:00000001:2.0:1775768243.890042:0:36:0:(genops.c:811:class_conn2export()) Process entered 00020000:00000002:3.0:1775768243.890045:0:13471:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00000020:00000040:2.0:1775768243.890052:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00020000:00000001:3.0:1775768243.890053:0:13471:0:(lov_io.c:155:lov_sub_get()) Process entered 00000100:00000001:1.0:1775768243.890054:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00020000:00000001:3.0:1775768243.890054:0:13471:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000100:00000040:1.0:1775768243.890058:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dff480 x1862022294291840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768268 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1775768243.890058:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000001:3.0:1775768243.890058:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000040:2.0:1775768243.890062:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000010:3.0:1775768243.890063:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff99420693ecf0. 00000020:00000001:2.0:1775768243.890066:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000010:3.0:1775768243.890069:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcf400. 00000020:00000001:2.0:1775768243.890072:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000001:1.0:1775768243.890073:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1775768243.890074:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210dcf200. 00000100:00000001:1.0:1775768243.890077:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:2.0:1775768243.890079:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000010:3.0:1775768243.890079:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207981900. 00000100:00000001:1.0:1775768243.890082:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768243.890083:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000008:00000010:3.0:1775768243.890085:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994208b51650. 00000100:00000040:2.0:1775768243.890087:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000200:1.0:1775768243.890089:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768243.890092:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:1.0:1775768243.890093:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4300170045 00000020:00000040:2.0:1775768243.890096:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00000010:1.0:1775768243.890100:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba240. 00000100:00080000:2.0:1775768243.890100:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00020000:00000010:3.0:1775768243.890100:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994218000000. 00000100:00000001:2.0:1775768243.890105:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768243.890108:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1775768243.890109:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00008000:00000010:3.0:1775768243.890109:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994207e0ade8. 02000000:00000001:2.0:1775768243.890111:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000080:00000010:3.0:1775768243.890115:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99420541b0a8. 02000000:00000001:2.0:1775768243.890116:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768243.890118:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4300170045 00000100:00000001:2.0:1775768243.890120:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000080:00000010:3.0:1775768243.890120:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077c1da8. 00000008:00100000:2.0:1775768243.890123:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00000010:1.0:1775768243.890124:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba3c0. 00000020:00000010:3.0:1775768243.890126:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dce000. 00000008:00000001:2.0:1775768243.890127:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768243.890130:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1775768243.890130:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:3.0:1775768243.890131:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942086f8160. 00000100:00000001:1.0:1775768243.890136:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000010:3.0:1775768243.890137:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994207e8ed00. 00000100:00000010:2.0:1775768243.890138:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfed80. 00000100:00000001:1.0:1775768243.890142:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775768243.890143:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00008000:00000010:3.0:1775768243.890143:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994209270ef0. 00000100:00000001:1.0:1775768243.890147:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 02000000:00000001:2.0:1775768243.890148:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000080:00000010:3.0:1775768243.890149:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942090e3960. 02000000:00000001:2.0:1775768243.890151:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768243.890153:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107590089976 : -112966119461640 : ffff99420693ecf8) 00000100:00000001:2.0:1775768243.890175:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775768243.890178:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000020:00001000:3.0:1775768243.890179:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff99420693ecf0 00000100:00100000:1.0:1775768243.890180:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768243/real 1775768243] req@ffff994207dff480 x1862022294291840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1775768243.890181:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768243.890182:0:13471:0:(cl_object.c:151:cl_object_top()) ffff9942078a75f8 -> ffff9942078a7f58 00000020:00000001:3.0:1775768243.890187:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 02000000:00000010:2.0:1775768243.890189:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1ec00. 00000020:00000001:3.0:1775768243.890191:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:3.0:1775768243.890195:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00000001:2.0:1775768243.890196:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768243.890197:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1775768243.890199:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00020000:00000001:3.0:1775768243.890201:0:13471:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1775768243.890202:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00100000:2.0:1775768243.890203:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfed80 x1862022294292096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:1.0:1775768243.890206:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c738. 00020000:00000001:3.0:1775768243.890206:0:13471:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107612435560 : -112966097116056 : ffff994207e8e468) 00000020:00000001:3.0:1775768243.890211:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000020:00000001:3.0:1775768243.890214:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768243.890216:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00020000:00000001:3.0:1775768243.890217:0:13471:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768243.890219:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf65d8 00000100:00000001:2.0:1775768243.890220:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768243.890221:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775768243.890223:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf65d8. 00000020:00000001:3.0:1775768243.890225:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00000100:00000040:2.0:1775768243.890227:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb9b9eae] to pc [ptlrpcd_rcv+-1] req@ffff994207dfed80 x1862022294292096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:3.0:1775768243.890228:0:13471:0:(lov_io.c:1108:lov_io_lock()) Process entered 00000100:00000001:1.0:1775768243.890229:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00020000:00000001:3.0:1775768243.890231:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775768243.890233:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:3.0:1775768243.890235:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000100:00000200:1.0:1775768243.890236:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dff480 x1862022294291840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775768243.890238:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768243.890243:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768243.890247:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768243.890251:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768243.890251:0:13471:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768243.890256:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000001:3.0:1775768243.890256:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:3.0:1775768243.890258:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775768243.890260:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dff480 x1862022294291840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775768243.890263:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768243.890264:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1775768243.890268:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000020:00000001:3.0:1775768243.890268:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00020000:00000001:3.0:1775768243.890271:0:13471:0:(lov_io.c:1114:lov_io_start()) Process entered 00000100:00080000:2.0:1775768243.890272:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768243.890273:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00020000:00000001:3.0:1775768243.890273:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000001:1.0:1775768243.890275:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1775768243.890276:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00000100:00000001:2.0:1775768243.890278:0:36:0:(import.c:484:import_select_connection()) Process entered 00000008:00000001:3.0:1775768243.890279:0:13471:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000100:00000001:1.0:1775768243.890280:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000008:00000001:3.0:1775768243.890282:0:13471:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000100:00000001:1.0:1775768243.890283:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1775768243.890283:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5496 00000008:00000001:3.0:1775768243.890288:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000100:00100000:1.0:1775768243.890289:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dff480 x1862022294291840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1775768243.890291:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00000020:3.0:1775768243.890291:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:3.0:1775768243.890294:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1775768243.890298:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000008:00000001:3.0:1775768243.890300:0:13471:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000008:00000001:3.0:1775768243.890303:0:13471:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000100:00000001:1.0:1775768243.890304:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1775768243.890304:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000008:00000001:3.0:1775768243.890305:0:13471:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107884688256 : -112965824863360 : ffff994218232380) 00000008:00000020:3.0:1775768243.890309:0:13471:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994218232380 ready 0|-|- wr 1|-|+ rd 0|- 9 in flight 00000100:00000040:2.0:1775768243.890310:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768243.890315:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000008:00000001:3.0:1775768243.890315:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000100:00000040:1.0:1775768243.890317:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dff480 x1862022294291840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768268 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000020:3.0:1775768243.890318:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000100:00000040:2.0:1775768243.890321:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000008:00000001:3.0:1775768243.890321:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1775768243.890324:0:13471:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000100:00000001:2.0:1775768243.890327:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000008:00000020:3.0:1775768243.890328:0:13471:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994210674fc0 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000100:00000001:1.0:1775768243.890332:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:2.0:1775768243.890333:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000008:00000001:3.0:1775768243.890335:0:13471:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768243.890337:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:2.0:1775768243.890337:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000008:00000001:3.0:1775768243.890337:0:13471:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768243.890341:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:1.0:1775768243.890342:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:3.0:1775768243.890342:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775768243.890343:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000100:00000001:1.0:1775768243.890344:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00020000:00000001:3.0:1775768243.890345:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768243.890348:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:1.0:1775768243.890349:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00020000:00000001:3.0:1775768243.890349:0:13471:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768243.890352:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:2.0:1775768243.890353:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:3.0:1775768243.890353:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768243.890355:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000020:00000001:3.0:1775768243.890356:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00020000:00000001:3.0:1775768243.890358:0:13471:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00000100:00000040:2.0:1775768243.890360:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00020000:00000001:3.0:1775768243.890361:0:13471:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000100:00080000:1.0:1775768243.890364:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000020:00000001:3.0:1775768243.890364:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00000100:00000001:2.0:1775768243.890365:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:3.0:1775768243.890367:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00000100:00000001:1.0:1775768243.890369:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00020000:00000001:3.0:1775768243.890369:0:13471:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768243.890370:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00020000:00000001:3.0:1775768243.890373:0:13471:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000100:00000001:2.0:1775768243.890374:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775768243.890375:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00000100:00000040:1.0:1775768243.890376:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dff480 x1862022294291840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768268 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775768243.890378:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000020:00000040:2.0:1775768243.890379:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00020000:00000001:3.0:1775768243.890380:0:13471:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00020000:00000001:3.0:1775768243.890383:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775768243.890385:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000100:00080000:2.0:1775768243.890386:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000020:00000001:3.0:1775768243.890387:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:3.0:1775768243.890389:0:13471:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768243.890393:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768243.890393:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768243.890397:0:13471:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 02000000:00000001:2.0:1775768243.890398:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000020:00000001:3.0:1775768243.890400:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000100:00100000:1.0:1775768243.890401:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294291840:192.168.202.156@tcp:250:kworker.0 02000000:00000001:2.0:1775768243.890402:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1775768243.890402:0:13471:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000020:00000001:3.0:1775768243.890403:0:13471:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000001:3.0:1775768243.890405:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 02000000:00000001:2.0:1775768243.890407:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768243.890408:0:13471:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00020000:00000001:3.0:1775768243.890410:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000001:1.0:1775768243.890411:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768243.890411:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000020:00000001:3.0:1775768243.890411:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000001:3.0:1775768243.890414:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000100:00000001:1.0:1775768243.890415:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000008:00100000:2.0:1775768243.890415:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00020000:00000001:3.0:1775768243.890416:0:13471:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768243.890419:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1775768243.890420:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768243.890421:0:13471:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000100:00000001:2.0:1775768243.890424:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768243.890424:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000020:00000001:3.0:1775768243.890427:0:13471:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775768243.890429:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfe300. 00000020:00000001:3.0:1775768243.890429:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00000100:00000040:1.0:1775768243.890431:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dff480 x1862022294291840/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768268 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:3.0:1775768243.890431:0:13471:0:(lov_io.c:675:lov_io_fini()) Process entered 00000020:00000040:2.0:1775768243.890434:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00020000:00000001:3.0:1775768243.890434:0:13471:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:3.0:1775768243.890436:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:3.0:1775768243.890438:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 02000000:00000001:2.0:1775768243.890440:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000020:00001000:3.0:1775768243.890440:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff99420693ecf0 02000000:00000001:2.0:1775768243.890444:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1775768243.890444:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210dcf200. 00000100:00000001:2.0:1775768243.890449:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1775768243.890450:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 00000020:00000010:3.0:1775768243.890450:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207981900. 02000000:00000001:2.0:1775768243.890452:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775768243.890454:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000008:00000010:3.0:1775768243.890454:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994208b51650. 02000000:00000001:2.0:1775768243.890455:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775768243.890457:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543c400. 00020000:00000010:3.0:1775768243.890459:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994218000000. 02000000:00000001:1.0:1775768243.890461:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000010:2.0:1775768243.890461:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1fc00. 00008000:00000010:3.0:1775768243.890462:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994207e0ade8. 00000020:00000001:1.0:1775768243.890464:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768243.890467:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000100:00000001:2.0:1775768243.890467:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:3.0:1775768243.890468:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99420541b0a8. 00000020:00000001:1.0:1775768243.890469:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768243.890473:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1f400. 00000080:00000010:3.0:1775768243.890473:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077c1da8. 00000100:00100000:2.0:1775768243.890478:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfe300 x1862022294292224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000010:3.0:1775768243.890478:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcf400. 02000000:00000001:1.0:1775768243.890479:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768243.890482:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000008:00000010:3.0:1775768243.890483:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942086f8160. 00000100:00000010:1.0:1775768243.890485:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dff480. 00020000:00000010:3.0:1775768243.890488:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994207e8ed00. 00000100:00000001:1.0:1775768243.890491:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00008000:00000010:3.0:1775768243.890492:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994209270ef0. 00000100:00000001:1.0:1775768243.890494:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00000010:3.0:1775768243.890497:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942090e3960. 00000100:00000001:1.0:1775768243.890499:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768243.890499:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000020:00000010:3.0:1775768243.890500:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dce000. 00000100:00000001:2.0:1775768243.890503:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1775768243.890505:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff99420693ecf0. 00000100:00000001:1.0:1775768243.890508:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00020000:00000001:3.0:1775768243.890508:0:13471:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00000100:00000001:1.0:1775768243.890512:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768243.890512:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c661fef] to pc [ptlrpcd_rcv+-1] req@ffff994207dfe300 x1862022294292224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:3.0:1775768243.890512:0:13471:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:3.0:1775768243.890514:0:13471:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000080:00200000:3.0:1775768243.890517:0:13471:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00000001:1.0:1775768243.890518:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768243.890523:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000080:00000001:3.0:1775768243.890523:0:13471:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000100:00000001:1.0:1775768243.890526:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1775768243.890527:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:3.0:1775768243.890530:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff99420693e000 00000020:00000010:3.0:1775768243.890532:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210dce800. 00000100:00000001:2.0:1775768243.890533:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1775768243.890546:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207981e40. 00000008:00000010:3.0:1775768243.890548:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994208b50000. 00000100:00000040:1.0:1775768243.890549:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfce00 x1862022294291968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000010:3.0:1775768243.890552:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994218000300. 00000100:00080000:2.0:1775768243.890553:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00008000:00000010:3.0:1775768243.890556:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994207e08e20. 00000080:00000010:3.0:1775768243.890560:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99420541b1f8. 00000100:00080000:2.0:1775768243.890564:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000080:00000010:3.0:1775768243.890564:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077c0248. 00000100:00000001:2.0:1775768243.890567:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775768243.890569:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000010:3.0:1775768243.890569:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcea00. 00000100:00080000:2.0:1775768243.890570:0:36:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768243.890572:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:3.0:1775768243.890573:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942086f8420. 00000100:00000001:2.0:1775768243.890574:0:36:0:(import.c:484:import_select_connection()) Process entered 00020000:00000010:3.0:1775768243.890577:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994207e8e400. 02000000:00000001:1.0:1775768243.890579:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00008000:00000010:3.0:1775768243.890580:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994209270a90. 00000100:00080000:2.0:1775768243.890582:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5496 02000000:00000001:1.0:1775768243.890583:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000080:00000010:3.0:1775768243.890583:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942090e33c0. 02000000:00000001:1.0:1775768243.890588:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1775768243.890588:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcf800. 00000400:00000200:2.0:1775768243.890592:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000010:3.0:1775768243.890593:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff99420693e000. 00000100:00100000:1.0:1775768243.890596:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294291968:192.168.202.156@tcp:38:kworker.0 00000080:00000001:3.0:1775768243.890597:0:13471:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1775768243.890598:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000080:00000001:3.0:1775768243.890601:0:13471:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768243.890604:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00080000:2.0:1775768243.890604:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000080:00000001:3.0:1775768243.890606:0:13471:0:(rw.c:1481:ll_writepages()) Process entered 02000000:00000001:1.0:1775768243.890607:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000080:00000001:3.0:1775768243.890608:0:13471:0:(file.c:5250:cl_sync_file_range()) Process entered 02000000:00000001:1.0:1775768243.890609:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768243.890610:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000001:3.0:1775768243.890610:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 02000000:00000001:1.0:1775768243.890613:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1775768243.890614:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000010:3.0:1775768243.890614:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff99420693e000. 02000000:00000010:1.0:1775768243.890616:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543c400. 02000000:00000001:1.0:1775768243.890619:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768243.890619:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000010:3.0:1775768243.890619:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcf800. 00000020:00000010:3.0:1775768243.890623:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210dcea00. 00000100:00000001:2.0:1775768243.890625:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000010:3.0:1775768243.890626:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207981e40. 00000400:00000200:1.0:1775768243.890627:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:2.0:1775768243.890631:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000400:00000010:1.0:1775768243.890632:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c738. 00000008:00000010:3.0:1775768243.890632:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994208b50000. 00000020:00000040:2.0:1775768243.890633:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:2.0:1775768243.890637:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000010:1.0:1775768243.890638:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf65d8. 00020000:00000010:3.0:1775768243.890638:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994218000300. 00000020:00000040:2.0:1775768243.890639:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00008000:00000010:3.0:1775768243.890642:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994207e08e20. 00000100:00000200:1.0:1775768243.890643:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294291968, portal 10 00000020:00000001:2.0:1775768243.890644:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000080:00000010:3.0:1775768243.890646:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99420541b1f8. 00000100:00000001:1.0:1775768243.890647:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000020:00000001:2.0:1775768243.890650:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000001:1.0:1775768243.890651:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611840000 : -112966097711616 : ffff994207dfce00) 00000080:00000010:3.0:1775768243.890651:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077c0248. 00000100:00000040:2.0:1775768243.890657:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000010:3.0:1775768243.890657:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dce800. 00000100:00000040:1.0:1775768243.890660:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfce00 x1862022294291968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768268 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000010:3.0:1775768243.890662:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942086f8420. 00000100:00000001:2.0:1775768243.890663:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768243.890668:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00020000:00000010:3.0:1775768243.890668:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994207e8e400. 00000100:00000001:1.0:1775768243.890673:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1775768243.890674:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00008000:00000010:3.0:1775768243.890674:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994209270a90. 00000100:00000040:1.0:1775768243.890679:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000020:00000040:2.0:1775768243.890680:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000080:00000010:3.0:1775768243.890680:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942090e33c0. 00000400:00000010:1.0:1775768243.890682:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000100:00000200:1.0:1775768243.890686:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294291968, offset 0 00000100:00080000:2.0:1775768243.890686:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000020:00000001:3.0:1775768243.890686:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107590086664 : -112966119464952 : ffff99420693e008) 00000020:00001000:3.0:1775768243.890691:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff99420693e000 00000100:00000001:2.0:1775768243.890692:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768243.890697:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 02000000:00000001:2.0:1775768243.890697:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000020:00000001:3.0:1775768243.890697:0:13471:0:(cl_object.c:151:cl_object_top()) ffff99420791a850 -> ffff99420791a850 02000000:00000001:2.0:1775768243.890701:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1775768243.890701:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:3.0:1775768243.890705:0:13471:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000400:00000200:1.0:1775768243.890707:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:2.0:1775768243.890707:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000080:00200000:3.0:1775768243.890708:0:13471:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017f:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000100:00000001:2.0:1775768243.890711:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000020:00000001:3.0:1775768243.890712:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000008:00100000:2.0:1775768243.890714:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000020:00000001:3.0:1775768243.890715:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000400:00000200:1.0:1775768243.890716:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000080:00000001:3.0:1775768243.890717:0:13471:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1775768243.890721:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:3.0:1775768243.890722:0:13471:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017f:0x0]io ffff9942077c02a8 type 6 ignore/verify layout 1/0 00000400:00000200:1.0:1775768243.890724:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1775768243.890725:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:3.0:1775768243.890728:0:13471:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207dfbd90) owner 0000000000000000/ffff994220080000 00000400:00000200:1.0:1775768243.890730:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000100:00000010:2.0:1775768243.890730:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfc700. 00020000:00000001:3.0:1775768243.890731:0:13471:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:3.0:1775768243.890733:0:13471:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00000400:00000200:1.0:1775768243.890736:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000020:00000040:2.0:1775768243.890736:0:36:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00020000:00000001:3.0:1775768243.890736:0:13471:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00020000:00000001:3.0:1775768243.890738:0:13471:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768243.890741:0:13471:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1775768243.890743:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 02000000:00000001:2.0:1775768243.890743:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00020000:00000001:3.0:1775768243.890744:0:13471:0:(lov_io.c:203:lov_io_subio_init()) Process entered 02000000:00000001:2.0:1775768243.890747:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768243.890747:0:13471:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768243.890750:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225454:225454:256:4294967295] 192.168.202.156@tcp LPNI seq info [225450:225450:8:4294967295] 00000100:00000001:2.0:1775768243.890752:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000020:00000001:3.0:1775768243.890752:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:3.0:1775768243.890754:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 02000000:00000001:2.0:1775768243.890756:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00020000:00000001:3.0:1775768243.890756:0:13471:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00000400:00000200:1.0:1775768243.890757:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000002:3.0:1775768243.890759:0:13471:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207dfbd90) owner 0000000000000000/ffff994220080000 02000000:00000001:2.0:1775768243.890760:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768243.890761:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768243.890766:0:13471:0:(cl_io.c:823:cl_io_loop()) Process entered 02000000:00000010:2.0:1775768243.890767:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1f800. 00000020:00000001:3.0:1775768243.890768:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000400:00000200:1.0:1775768243.890769:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00020000:00000001:3.0:1775768243.890771:0:13471:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00000100:00000001:2.0:1775768243.890773:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00020000:00200000:3.0:1775768243.890775:0:13471:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000800:00000200:1.0:1775768243.890778:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00020000:00000002:3.0:1775768243.890779:0:13471:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00000100:00100000:2.0:1775768243.890782:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfc700 x1862022294292352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000010:1.0:1775768243.890783:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541300. 00020000:00000001:3.0:1775768243.890783:0:13471:0:(lov_io.c:155:lov_sub_get()) Process entered 00020000:00000001:3.0:1775768243.890786:0:13471:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000020:00000001:3.0:1775768243.890789:0:13471:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000800:00000200:1.0:1775768243.890793:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000010:3.0:1775768243.890794:0:13471:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff99420693ecf0. 00000800:00000200:1.0:1775768243.890797:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000020:00000010:3.0:1775768243.890800:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dce000. 00000100:00000001:2.0:1775768243.890801:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000200:1.1:1775768243.890804:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000010:3.0:1775768243.890805:0:13471:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210dcf400. 00000100:00000001:2.0:1775768243.890806:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1775768243.890810:0:13471:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207981900. 00000800:00000200:1.1:1775768243.890810:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000100:00000040:2.0:1775768243.890814:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a176dd1] to pc [ptlrpcd_rcv+-1] req@ffff994207dfc700 x1862022294292352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000010:3.0:1775768243.890815:0:13471:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994208b51650. 00000800:00000100:1.0:1775768243.890820:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00020000:00000010:3.0:1775768243.890821:0:13471:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994218000000. 00000800:00000010:1.0:1775768243.890824:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541300. 00008000:00000010:3.0:1775768243.890825:0:13471:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994207e0ade8. 00000400:00000200:1.0:1775768243.890831:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000080:00000010:3.0:1775768243.890831:0:13471:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99420541b0a8. 00000100:00000001:2.0:1775768243.890832:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:3.0:1775768243.890833:0:13471:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077c1da8. 00000020:00000010:3.0:1775768243.890838:0:13471:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcf200. 00000400:00000200:1.0:1775768243.890839:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000100:00000040:2.0:1775768243.890840:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5506) 00000008:00000010:3.0:1775768243.890841:0:13471:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942086f8160. 00000400:00000200:1.0:1775768243.890844:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00020000:00000010:3.0:1775768243.890845:0:13471:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994207e8ed00. 00000400:00000010:1.0:1775768243.890847:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00008000:00000010:3.0:1775768243.890848:0:13471:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994209270ef0. 00000080:00000010:3.0:1775768243.890851:0:13471:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942090e3960. 00000100:00000001:1.0:1775768243.890852:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:3.0:1775768243.890853:0:13471:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107590089976 : -112966119461640 : ffff99420693ecf8) 00000100:00000200:1.0:1775768243.890857:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfce00 x1862022294291968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768268 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00001000:3.0:1775768243.890857:0:13471:0:(cl_object.c:831:cl_env_get()) 1@ffff99420693ecf0 00000020:00000001:3.0:1775768243.890861:0:13471:0:(cl_object.c:151:cl_object_top()) ffff99420791a210 -> ffff99420791a850 00000020:00000001:3.0:1775768243.890865:0:13471:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:3.0:1775768243.890867:0:13471:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:3.0:1775768243.890870:0:13471:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00000001:1.0:1775768243.890871:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000020:00000001:3.0:1775768243.890872:0:13471:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768243.890874:0:13471:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768243.890876:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfce00 x1862022294291968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768268 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:3.0:1775768243.890877:0:13471:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107612435560 : -112966097116056 : ffff994207e8e468) 00000020:00000001:3.0:1775768243.890881:0:13471:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000020:00000001:3.0:1775768243.890883:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768243.890887:0:13471:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768243.890889:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768243.890890:0:13471:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768243.890892:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:3.0:1775768243.890893:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00020000:00000001:3.0:1775768243.890896:0:13471:0:(lov_io.c:1108:lov_io_lock()) Process entered 00020000:00000001:3.0:1775768243.890897:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775768243.890898:0:13471:0:(cl_io.c:314:cl_io_lock()) Process entered 00000100:00000001:1.0:1775768243.890899:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768243.890900:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:3.0:1775768243.890903:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768243.890904:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000020:00000001:3.0:1775768243.890904:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768243.890905:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768243.890908:0:13471:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768243.890911:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:3.0:1775768243.890912:0:13471:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:3.0:1775768243.890914:0:13471:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768243.890915:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300168723 00000020:00000001:3.0:1775768243.890915:0:13471:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768243.890918:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300170045 00000020:00000001:3.0:1775768243.890918:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00020000:00000001:3.0:1775768243.890919:0:13471:0:(lov_io.c:1114:lov_io_start()) Process entered 00020000:00000001:3.0:1775768243.890921:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000001:1.0:1775768243.890922:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300168723 00000020:00000001:3.0:1775768243.890922:0:13471:0:(cl_io.c:528:cl_io_start()) Process entered 00000100:00000001:1.0:1775768243.890925:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300170045 00000008:00000001:3.0:1775768243.890925:0:13471:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000008:00000001:3.0:1775768243.890927:0:13471:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000100:00000001:1.0:1775768243.890928:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00000001:3.0:1775768243.890929:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000100:00000001:1.0:1775768243.890930:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000020:3.0:1775768243.890931:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000100:00000001:1.0:1775768243.890932:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000008:00000001:3.0:1775768243.890933:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1775768243.890935:0:13471:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000008:00000001:3.0:1775768243.890937:0:13471:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000100:00100000:1.0:1775768243.890938:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768243/real 1775768243] req@ffff994207dfce00 x1862022294291968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1775768243.890939:0:13471:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107754938304 : -112965954613312 : ffff994210674fc0) 00000008:00000020:3.0:1775768243.890942:0:13471:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994210674fc0 ready 0|-|- wr 1|-|+ rd 0|- 9 in flight 00000008:00000001:3.0:1775768243.890949:0:13471:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:3.0:1775768243.890950:0:13471:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000400:00000001:1.0:1775768243.890953:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000008:00000001:3.0:1775768243.890953:0:13471:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1775768243.890954:0:13471:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000400:00000001:1.0:1775768243.890955:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000008:00000020:3.0:1775768243.890956:0:13471:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994218232380 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000400:00000010:1.0:1775768243.890957:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c738. 00000400:00000200:1.0:1775768243.890961:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf65d8 00000008:00000001:3.0:1775768243.890961:0:13471:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:1.0:1775768243.890964:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf65d8. 00000008:00000001:3.0:1775768243.890964:0:13471:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768243.890968:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768243.890969:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00020000:00000001:3.0:1775768243.890971:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775768243.890972:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfce00 x1862022294291968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:3.0:1775768243.890973:0:13471:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768243.890977:0:13471:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768243.890980:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00020000:00000001:3.0:1775768243.890982:0:13471:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00020000:00000001:3.0:1775768243.890983:0:13471:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000020:00000001:3.0:1775768243.890983:0:13471:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000001:3.0:1775768243.890984:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00020000:00000001:3.0:1775768243.890984:0:13471:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768243.890985:0:13471:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000020:00000001:3.0:1775768243.890986:0:13471:0:(cl_io.c:563:cl_io_end()) Process leaving 00000020:00000001:3.0:1775768243.890987:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00020000:00000001:3.0:1775768243.890988:0:13471:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00020000:00000001:3.0:1775768243.890989:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775768243.890990:0:13471:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000020:00000001:3.0:1775768243.890992:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:3.0:1775768243.890994:0:13471:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768243.890997:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768243.890998:0:13471:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000100:00000200:1.0:1775768243.891000:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfce00 x1862022294291968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775768243.891000:0:13471:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:3.0:1775768243.891002:0:13471:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000020:00000001:3.0:1775768243.891004:0:13471:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000001:3.0:1775768243.891007:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:3.0:1775768243.891009:0:13471:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00020000:00000001:3.0:1775768243.891011:0:13471:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000001:1.0:1775768243.891014:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:3.0:1775768243.891014:0:13471:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000001:3.0:1775768243.891015:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000100:00000001:1.0:1775768243.891017:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00020000:00000001:3.0:1775768243.891018:0:13471:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768243.891020:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00020000:00000001:3.0:1775768243.891021:0:13471:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768243.891023:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00020000:00000001:3.0:1775768243.891024:0:13471:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000020:00000001:3.0:1775768243.891026:0:13471:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000100:00100000:1.0:1775768243.891029:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfce00 x1862022294291968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775768243.891029:0:13471:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768243.891032:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:3.0:1775768243.891034:0:13471:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:3.0:1775768243.891036:0:13471:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:3.0:1775768243.891037:0:13471:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:3.0:1775768243.891038:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:3.0:1775768243.891040:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff99420693ecf0 00000020:00000010:3.0:1775768243.891044:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210dcf400. 00000100:00000001:1.0:1775768243.891045:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:3.0:1775768243.891049:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207981900. 00000100:00000040:1.0:1775768243.891051:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfce00 x1862022294291968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768268 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000010:3.0:1775768243.891053:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994208b51650. 00020000:00000010:3.0:1775768243.891058:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994218000000. 00008000:00000010:3.0:1775768243.891063:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994207e0ade8. 00000100:00000001:1.0:1775768243.891064:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768243.891068:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000080:00000010:3.0:1775768243.891068:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99420541b0a8. 00000100:00000001:1.0:1775768243.891072:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000080:00000010:3.0:1775768243.891072:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077c1da8. 00000100:00000001:1.0:1775768243.891074:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768243.891077:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000020:00000010:3.0:1775768243.891077:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dce000. 00000100:00000001:1.0:1775768243.891081:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00000010:3.0:1775768243.891081:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942086f8160. 00000100:00080000:1.0:1775768243.891085:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00020000:00000010:3.0:1775768243.891086:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994207e8ed00. 00008000:00000010:3.0:1775768243.891090:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994209270ef0. 00000100:00080000:1.0:1775768243.891092:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000080:00000010:3.0:1775768243.891093:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942090e3960. 00000100:00000001:1.0:1775768243.891094:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000010:3.0:1775768243.891096:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcf200. 00000020:00000010:3.0:1775768243.891099:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff99420693ecf0. 00020000:00000001:3.0:1775768243.891101:0:13471:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00000100:00000040:1.0:1775768243.891103:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfce00 x1862022294291968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768268 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:3.0:1775768243.891103:0:13471:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:3.0:1775768243.891105:0:13471:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000080:00200000:3.0:1775768243.891107:0:13471:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017f:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:3.0:1775768243.891113:0:13471:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:3.0:1775768243.891114:0:13471:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:3.0:1775768243.891117:0:13471:0:(cl_object.c:925:cl_env_put()) 1@ffff99420693e000 00000020:00000010:3.0:1775768243.891120:0:13471:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210dcea00. 00000100:00100000:1.0:1775768243.891122:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294291968:192.168.202.156@tcp:38:kworker.0 00000020:00000010:3.0:1775768243.891123:0:13471:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207981e40. 00000008:00000010:3.0:1775768243.891126:0:13471:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994208b50000. 00020000:00000010:3.0:1775768243.891130:0:13471:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994218000300. 00000100:00000001:1.0:1775768243.891131:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00008000:00000010:3.0:1775768243.891133:0:13471:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994207e08e20. 00000080:00000010:3.0:1775768243.891135:0:13471:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99420541b1f8. 00000100:00000040:1.0:1775768243.891137:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfed80 x1862022294292096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000010:3.0:1775768243.891137:0:13471:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077c0248. 00000020:00000010:3.0:1775768243.891140:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcf800. 00000008:00000010:3.0:1775768243.891143:0:13471:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942086f8420. 00020000:00000010:3.0:1775768243.891146:0:13471:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994207e8e400. 00008000:00000010:3.0:1775768243.891149:0:13471:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994209270a90. 00000100:00000001:1.0:1775768243.891151:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000080:00000010:3.0:1775768243.891152:0:13471:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942090e33c0. 00000100:00000001:1.0:1775768243.891171:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768243.891174:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768243.891176:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1775768243.891176:0:13471:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dce800. 00000020:00000010:3.0:1775768243.891180:0:13471:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff99420693e000. 02000000:00000001:1.0:1775768243.891181:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:3.0:1775768243.891182:0:13471:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:3.0:1775768243.891184:0:13471:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768243.891186:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294292096:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768243.891191:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768243.891193:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768243.891195:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768243.891197:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768243.891201:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543d800. 02000000:00000001:1.0:1775768243.891203:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768243.891207:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768243.891209:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c738. 00000400:00000010:1.0:1775768243.891213:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf65d8. 00000100:00000200:1.0:1775768243.891216:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294292096, portal 10 00000100:00000001:1.0:1775768243.891217:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768243.891219:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611848064 : -112966097703552 : ffff994207dfed80) 00000100:00000040:1.0:1775768243.891224:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfed80 x1862022294292096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768268 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768243.891229:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768243.891231:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768243.891234:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000100:00000200:1.0:1775768243.891236:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294292096, offset 0 00000400:00000200:1.0:1775768243.891240:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768243.891244:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768243.891247:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768243.891250:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768243.891252:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768243.891254:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768243.891257:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768243.891259:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225455:225455:256:4294967295] 192.168.202.156@tcp LPNI seq info [225451:225451:8:4294967295] 00000400:00000200:1.0:1775768243.891263:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768243.891268:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768243.891271:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768243.891273:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541300. 00000800:00000200:1.0:1775768243.891276:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768243.891277:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775768243.891280:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768243.891282:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1775768243.891284:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768243.891285:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541300. 00000400:00000200:1.0:1775768243.891287:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768243.891289:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768243.891291:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000400:00000010:1.0:1775768243.891292:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000100:00000001:1.0:1775768243.891294:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768243.891296:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfed80 x1862022294292096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768268 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768243.891301:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768243.891302:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfed80 x1862022294292096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768268 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768243.891306:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768243.891307:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768243.891309:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768243.891311:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:1.0:1775768243.891313:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768243.891314:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300168716 00000100:00000001:1.0:1775768243.891316:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300170045 00000100:00000001:1.0:1775768243.891317:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300168716 00000100:00000001:1.0:1775768243.891318:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300170045 00000100:00000001:1.0:1775768243.891319:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768243.891320:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768243.891321:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768243.891323:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768243/real 1775768243] req@ffff994207dfed80 x1862022294292096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768243.891327:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768243.891328:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768243.891329:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c738. 00000400:00000200:1.0:1775768243.891331:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf65d8 00000400:00000010:1.0:1775768243.891332:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf65d8. 00000100:00000001:1.0:1775768243.891333:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768243.891334:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfed80 x1862022294292096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768243.891339:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfed80 x1862022294292096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768243.891342:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768243.891343:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768243.891344:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768243.891345:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768243.891346:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfed80 x1862022294292096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768243.891350:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768243.891352:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfed80 x1862022294292096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768268 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768243.891355:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768243.891357:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768243.891358:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768243.891359:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768243.891360:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768243.891362:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768243.891363:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768243.891365:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768243.891366:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768243.891368:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfed80 x1862022294292096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768268 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768243.891372:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294292096:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768243.891375:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768243.891376:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768243.891377:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfce00 x1862022294291968/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768268 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768243.891381:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768243.891381:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768243.891383:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543c400. 02000000:00000001:1.0:1775768243.891384:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768243.891384:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768243.891385:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:1.0:1775768243.891387:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768243.891388:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1f000. 02000000:00000001:1.0:1775768243.891390:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768243.891392:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768243.891393:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfce00. 00000100:00000001:1.0:1775768243.891395:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768243.891396:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768243.891397:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768243.891400:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfed80 x1862022294292096/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768268 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768243.891411:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768243.891412:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768243.891413:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543d800. 02000000:00000001:1.0:1775768243.891415:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768243.891416:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768243.891417:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:1.0:1775768243.891419:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768243.891420:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1ec00. 02000000:00000001:1.0:1775768243.891423:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768243.891424:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768243.891425:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfed80. 00000100:00000001:1.0:1775768243.891427:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768243.891428:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768243.891430:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768243.891433:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768243.891434:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768243.891436:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768243.891437:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768243.891439:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768243.891442:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfe300 x1862022294292224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768243.891448:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768243.891449:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768243.891451:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768243.891452:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768243.891453:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768243.891455:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294292224:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768243.891457:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768243.891458:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768243.891459:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768243.891460:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768243.891462:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543d800. 02000000:00000001:1.0:1775768243.891463:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768243.891465:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768243.891467:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c738. 00000400:00000010:1.0:1775768243.891469:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf65d8. 00000100:00000200:1.0:1775768243.891471:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294292224, portal 4 00000100:00000001:1.0:1775768243.891472:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768243.891472:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611845376 : -112966097706240 : ffff994207dfe300) 00000100:00000040:1.0:1775768243.891476:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfe300 x1862022294292224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768268 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768243.891479:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768243.891481:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768243.891482:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000100:00000200:1.0:1775768243.891483:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294292224, offset 0 00000400:00000200:1.0:1775768243.891485:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768243.891488:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768243.891491:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768243.891493:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768243.891496:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768243.891498:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768243.891500:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768243.891502:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225456:225456:256:4294967295] 192.168.202.156@tcp LPNI seq info [225452:225452:8:4294967295] 00000400:00000200:1.0:1775768243.891506:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768243.891512:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768243.891516:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768243.891520:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541300. 00000800:00000200:1.0:1775768243.891523:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768243.891526:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775768243.891530:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768243.891532:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1775768243.891545:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768243.891547:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541300. 00000400:00000200:1.0:1775768243.891549:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768243.891551:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768243.891553:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000400:00000010:1.0:1775768243.891554:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000100:00000001:1.0:1775768243.891556:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768243.891558:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfe300 x1862022294292224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768268 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768243.891563:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768243.891565:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfe300 x1862022294292224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768268 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768243.891568:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768243.891569:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768243.891571:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768243.891573:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:1.0:1775768243.891575:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768243.891576:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300168739 00000100:00000001:1.0:1775768243.891578:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300170045 00000100:00000001:1.0:1775768243.891579:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300168739 00000100:00000001:1.0:1775768243.891580:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300170045 00000100:00000001:1.0:1775768243.891581:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768243.891582:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768243.891584:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768243.891585:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768243/real 1775768243] req@ffff994207dfe300 x1862022294292224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768243.891590:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768243.891591:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768243.891592:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c738. 00000400:00000200:1.0:1775768243.891593:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf65d8 00000400:00000010:1.0:1775768243.891594:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf65d8. 00000100:00000001:1.0:1775768243.891595:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768243.891596:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfe300 x1862022294292224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768243.891601:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfe300 x1862022294292224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768243.891605:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768243.891606:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768243.891607:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768243.891608:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768243.891610:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfe300 x1862022294292224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768243.891613:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768243.891615:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfe300 x1862022294292224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768268 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768243.891618:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768243.891620:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768243.891620:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768243.891621:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768243.891622:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768243.891624:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768243.891625:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768243.891627:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768243.891628:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768243.891630:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfe300 x1862022294292224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768268 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768243.891634:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294292224:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768243.891636:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768243.891637:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfc700 x1862022294292352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768243.891641:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768243.891642:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768243.891643:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768243.891645:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768243.891646:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768243.891647:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294292352:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768243.891649:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768243.891650:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768243.891651:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768243.891652:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768243.891654:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543c400. 02000000:00000001:1.0:1775768243.891655:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768243.891656:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768243.891658:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c738. 00000400:00000010:1.0:1775768243.891659:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf65d8. 00000100:00000200:1.0:1775768243.891661:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294292352, portal 4 00000100:00000001:1.0:1775768243.891662:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768243.891663:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611838208 : -112966097713408 : ffff994207dfc700) 00000100:00000040:1.0:1775768243.891666:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfc700 x1862022294292352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768268 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768243.891669:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768243.891671:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768243.891672:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000100:00000200:1.0:1775768243.891673:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294292352, offset 0 00000400:00000200:1.0:1775768243.891675:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768243.891677:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768243.891679:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768243.891680:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768243.891682:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768243.891683:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768243.891685:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768243.891687:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225457:225457:256:4294967295] 192.168.202.156@tcp LPNI seq info [225453:225453:8:4294967295] 00000400:00000200:1.0:1775768243.891689:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768243.891693:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768243.891698:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768243.891701:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541300. 00000800:00000200:1.0:1775768243.891704:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768243.891706:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775768243.891710:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768243.891712:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1775768243.891716:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768243.891718:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541300. 00000400:00000200:1.0:1775768243.891720:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768243.891723:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768243.891726:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000400:00000010:1.0:1775768243.891727:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000100:00000001:1.0:1775768243.891730:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768243.891733:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfc700 x1862022294292352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768268 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768243.891740:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768243.891743:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfc700 x1862022294292352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768268 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768243.891747:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768243.891748:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768243.891750:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768243.891752:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:1.0:1775768243.891754:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768243.891755:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300168742 00000100:00000001:1.0:1775768243.891757:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300170045 00000100:00000001:1.0:1775768243.891758:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300168742 00000100:00000001:1.0:1775768243.891760:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300170045 00000100:00000001:1.0:1775768243.891761:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768243.891762:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768243.891763:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768243.891765:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768243/real 1775768243] req@ffff994207dfc700 x1862022294292352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768243.891769:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768243.891770:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768243.891771:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c738. 00000400:00000200:1.0:1775768243.891773:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf65d8 00000400:00000010:1.0:1775768243.891773:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf65d8. 00000100:00000001:1.0:1775768243.891774:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768243.891776:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfc700 x1862022294292352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768243.891780:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfc700 x1862022294292352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768243.891784:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768243.891785:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768243.891786:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768243.891786:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768243.891788:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfc700 x1862022294292352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768243.891791:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768243.891793:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfc700 x1862022294292352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768268 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768243.891797:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768243.891798:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768243.891798:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768243.891799:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768243.891800:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768243.891802:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768243.891803:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768243.891806:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768243.891807:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768243.891811:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfc700 x1862022294292352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768268 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768243.891818:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294292352:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768243.891821:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768243.891823:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768243.891826:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfe300 x1862022294292224/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768268 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768243.891831:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768243.891833:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768243.891834:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543d800. 02000000:00000001:1.0:1775768243.891837:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768243.891838:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768243.891839:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:1.0:1775768243.891841:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768243.891843:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1fc00. 02000000:00000001:1.0:1775768243.891845:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768243.891846:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768243.891847:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfe300. 00000100:00000001:1.0:1775768243.891849:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768243.891850:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768243.891852:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768243.891855:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfc700 x1862022294292352/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768268 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768243.891859:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768243.891860:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768243.891861:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543c400. 02000000:00000001:1.0:1775768243.891863:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768243.891864:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768243.891865:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:1.0:1775768243.891867:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768243.891868:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1f800. 02000000:00000001:1.0:1775768243.891870:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768243.891871:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768243.891872:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfc700. 00000100:00000001:1.0:1775768243.891874:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768243.891875:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768243.891876:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768243.891879:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768243.891881:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768243.891882:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768243.891883:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768243.953251:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768243.953257:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768243.953263:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768243.953265:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768243.953269:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768243.953270:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768243.953273:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768243.953274:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768243.953277:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768243.953278:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768243.953332:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768243.953336:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768243.953339:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775768244.017251:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768244.017256:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768244.017258:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768244.017263:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:0.0:1775768244.017266:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768244.017270:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768244.017272:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768244.017275:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768244.017277:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768244.913240:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768244.913240:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768244.913246:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768244.913247:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768244.913248:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768244.913250:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768244.913255:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775768244.913255:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:0.0:1775768244.913259:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768244.913260:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768244.913264:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768244.913264:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768244.913268:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768244.913271:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768244.913273:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768244.913276:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768244.913278:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768244.913284:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768244.913286:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768244.913289:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768244.913293:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768244.913296:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768244.913298:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768244.913303:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768244.913306:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768244.913309:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768244.913312:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768244.977249:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768244.977254:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768244.977258:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768244.977260:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768244.977262:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768244.977263:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768244.977265:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768244.977266:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768244.977268:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768244.977269:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768244.977316:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768244.977319:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768244.977322:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768245.041303:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768245.041313:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768245.041316:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768245.041322:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768245.041327:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768245.041331:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768245.041334:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768245.041337:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768245.041341:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768245.233292:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4300170045, at 4300170048 00000100:00000040:0.0:1775768245.233293:0:15036:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4300170045, at 4300170048 00000100:00000010:1.0:1775768245.233312:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cba3c0. 00000100:00000010:0.0:1775768245.233316:0:15036:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cba240. 00000100:00000001:0.0:1775768245.937247:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768245.937247:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768245.937247:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768245.937253:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768245.937254:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768245.937255:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768245.937255:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768245.937258:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768245.937258:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768245.937260:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775768245.937262:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:0.0:1775768245.937265:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1775768245.937265:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768245.937266:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768245.937269:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768245.937270:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768245.937270:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768245.937272:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768245.937275:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768245.937275:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768245.937279:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768245.937281:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768245.937284:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768245.937287:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768245.937287:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768245.937291:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768245.937294:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768246.001256:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768246.001261:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768246.001265:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768246.001266:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768246.001268:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768246.001269:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768246.001271:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768246.001272:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768246.001274:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768246.001275:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768246.001319:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768246.001323:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768246.001325:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768246.065229:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768246.065236:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768246.065238:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768246.065243:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768246.065246:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768246.065250:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768246.065253:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768246.065255:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768246.065258:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768246.961228:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768246.961228:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768246.961235:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768246.961235:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768246.961237:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768246.961238:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768246.961243:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:2.0:1775768246.961244:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768246.961244:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768246.961250:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768246.961250:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768246.961250:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768246.961252:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768246.961254:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768246.961255:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775768246.961256:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768246.961260:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768246.961261:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768246.961262:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768246.961264:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768246.961265:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768246.961268:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768246.961271:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768246.961274:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768246.961274:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768246.961277:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768246.961280:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768247.025253:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768247.025258:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768247.025262:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768247.025265:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768247.025266:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768247.025269:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768247.025271:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768247.025273:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768247.025274:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768247.025275:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768247.025278:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768247.025278:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768247.025280:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768247.025281:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768247.025282:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768247.025283:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768247.025284:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768247.025285:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768247.025288:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768247.025289:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768247.025289:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768247.025290:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768247.025291:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768247.025291:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768247.025292:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768247.025294:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768247.025296:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768247.025296:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768247.025297:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768247.025298:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768247.025299:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768247.025299:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768247.025300:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768247.025301:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768247.025343:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768247.025346:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768247.025348:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768247.089258:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768247.089262:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768247.089264:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768247.089267:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768247.089270:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768247.089273:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768247.089275:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768247.089277:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768247.089279:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768247.985261:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768247.985261:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768247.985261:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768247.985267:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768247.985268:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768247.985268:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768247.985269:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768247.985272:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768247.985272:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768247.985274:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:2.0:1775768247.985278:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775768247.985278:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768247.985279:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768247.985284:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768247.985284:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768247.985286:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768247.985289:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768247.985289:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768247.985290:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768247.985294:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768247.985298:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768247.985298:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768247.985302:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768247.985305:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768247.985305:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768247.985308:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768247.985310:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768248.049285:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768248.049291:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768248.049297:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768248.049299:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768248.049302:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768248.049304:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768248.049308:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768248.049309:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768248.049313:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768248.049315:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768248.049374:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768248.049378:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768248.049381:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768248.113268:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768248.113275:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768248.113277:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768248.113283:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768248.113287:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768248.113290:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768248.113293:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768248.113295:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768248.113298:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768249.009310:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768249.009315:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768249.009316:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768249.009319:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768249.009323:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768249.009326:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768249.009326:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:1.0:1775768249.009337:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768249.009337:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768249.009343:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768249.009343:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768249.009349:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768249.009349:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768249.009353:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768249.009357:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768249.009365:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768249.009368:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768249.009372:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768249.009517:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768249.009522:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775768249.009525:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1775768249.009526:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:3.0:1775768249.009529:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768249.009530:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768249.009533:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768249.009536:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768249.009536:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:0.0:1775768249.009541:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768249.009543:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775768249.009546:0:13471:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5501 00000100:00000001:0.0:1775768249.009547:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768249.009552:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768249.009555:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775768249.009564:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775768249.009575:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775768249.009580:0:13471:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775768249.009584:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768249.009588:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775768249.009591:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768249.009593:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775768249.009597:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775768249.009599:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:3.0:1775768249.009602:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775768249.009605:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:3.0:1775768249.009607:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:3.0:1775768249.009609:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:3.0:1775768249.009612:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768249.009615:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775768249.009616:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768249.009618:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775768249.009621:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:3.0:1775768249.009624:0:13471:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775768249.009627:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768249.009631:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775768249.009635:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775768249.009638:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768249.009641:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775768249.009643:0:13471:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775768249.009655:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8a80. 00000020:00000040:3.0:1775768249.009659:0:13471:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:3.0:1775768249.009662:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775768249.009665:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768249.009669:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775768249.009672:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775768249.009677:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775768249.009685:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2c00. 00000100:00000001:3.0:1775768249.009692:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768249.009700:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8a80 x1862022294292480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775768249.009708:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775768249.009712:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768249.009715:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006c3265af] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8a80 x1862022294292480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768249.009734:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768249.009737:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768249.009741:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775768249.009742:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775768249.009744:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775768249.009747:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775768249.009749:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5501 00000400:00000200:3.0:1775768249.009753:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775768249.009757:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775768249.009759:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775768249.009762:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768249.009764:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775768249.009764:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775768249.009765:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768249.009767:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:1.0:1775768249.009768:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1775768249.009769:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:1.0:1775768249.009770:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000040:3.0:1775768249.009770:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:3.0:1775768249.009772:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775768249.009775:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000100:00000040:1.0:1775768249.009777:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a8a80 x1862022294292480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775768249.009777:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:3.0:1775768249.009779:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:3.0:1775768249.009782:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768249.009784:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775768249.009785:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768249.009786:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768249.009786:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768249.009788:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 02000000:00000001:1.0:1775768249.009790:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:3.0:1775768249.009790:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00080000:3.0:1775768249.009792:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 02000000:00000001:1.0:1775768249.009793:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775768249.009794:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768249.009795:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775768249.009796:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768249.009799:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294292480:192.168.202.156@tcp:250:kworker.0 02000000:00000001:3.0:1775768249.009801:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775768249.009803:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768249.009805:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775768249.009806:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768249.009810:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768249.009812:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1775768249.009814:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1775768249.009815:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:3.0:1775768249.009816:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768249.009818:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775768249.009820:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543d800. 00000100:00000010:3.0:1775768249.009821:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aa300. 02000000:00000001:1.0:1775768249.009822:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775768249.009823:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:3.0:1775768249.009825:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775768249.009826:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768249.009828:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775768249.009828:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775768249.009830:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:1.0:1775768249.009832:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c738. 02000000:00000001:3.0:1775768249.009832:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775768249.009835:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2800. 00000100:00000001:3.0:1775768249.009838:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775768249.009839:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf65d8. 00000100:00100000:3.0:1775768249.009842:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aa300 x1862022294292608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:1.0:1775768249.009847:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294292480, portal 25 00000100:00000001:3.0:1775768249.009848:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775768249.009850:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775768249.009850:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768249.009851:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640855168 : -112966068696448 : ffff9942099a8a80) 00000100:00000040:3.0:1775768249.009854:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f33bdf15] to pc [ptlrpcd_rcv+-1] req@ffff9942099aa300 x1862022294292608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775768249.009857:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a8a80 x1862022294292480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768274 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768249.009862:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768249.009864:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1775768249.009864:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1775768249.009868:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000100:00080000:3.0:1775768249.009868:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775768249.009869:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000400:00000010:1.0:1775768249.009870:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000100:00080000:3.0:1775768249.009870:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:1.0:1775768249.009873:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294292480, offset 0 00000100:00000001:3.0:1775768249.009873:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775768249.009876:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5501 00000400:00000200:3.0:1775768249.009878:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768249.009881:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775768249.009881:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775768249.009883:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775768249.009886:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768249.009888:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775768249.009889:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000200:1.0:1775768249.009890:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775768249.009892:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775768249.009893:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775768249.009894:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000400:00000200:1.0:1775768249.009896:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000020:00000001:3.0:1775768249.009896:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775768249.009898:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:3.0:1775768249.009899:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:3.0:1775768249.009901:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000400:00000200:1.0:1775768249.009902:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:3.0:1775768249.009904:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000200:1.0:1775768249.009906:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000100:00000001:3.0:1775768249.009906:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775768249.009908:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768249.009909:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775768249.009910:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000400:00000200:1.0:1775768249.009911:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000100:00080000:3.0:1775768249.009913:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000400:00000200:1.0:1775768249.009914:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00000001:3.0:1775768249.009915:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768249.009916:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1775768249.009918:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225458:225458:256:4294967295] 192.168.202.156@tcp LPNI seq info [225454:225454:8:4294967295] 02000000:00000001:3.0:1775768249.009919:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775768249.009921:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768249.009923:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775768249.009923:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775768249.009925:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775768249.009927:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768249.009928:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775768249.009931:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ab800. 00000020:00000040:3.0:1775768249.009933:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000400:00000200:1.0:1775768249.009934:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 02000000:00000001:3.0:1775768249.009935:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775768249.009937:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768249.009938:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775768249.009939:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775768249.009941:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775768249.009944:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 02000000:00000010:3.0:1775768249.009948:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0000. 00000800:00000010:1.0:1775768249.009949:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541300. 00000100:00000001:3.0:1775768249.009951:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768249.009954:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ab800 x1862022294292736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.0:1775768249.009957:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768249.009961:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000100:00000001:3.0:1775768249.009961:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775768249.009963:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768249.009966:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000089df80c0] to pc [ptlrpcd_rcv+-1] req@ffff9942099ab800 x1862022294292736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.1:1775768249.009967:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768249.009971:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000100:00000001:3.0:1775768249.009973:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1775768249.009976:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000100:00080000:3.0:1775768249.009976:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000010:1.0:1775768249.009978:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541300. 00000100:00080000:3.0:1775768249.009979:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775768249.009981:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775768249.009982:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775768249.009984:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:1.0:1775768249.009986:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00080000:3.0:1775768249.009986:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5501 00000400:00000200:3.0:1775768249.009989:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768249.009991:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775768249.009992:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775768249.009994:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1775768249.009997:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000100:00000040:3.0:1775768249.009997:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768249.009999:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:1.0:1775768249.010000:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000100:00000040:3.0:1775768249.010001:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768249.010002:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:1.0:1775768249.010004:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:3.0:1775768249.010004:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775768249.010005:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:3.0:1775768249.010007:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000200:1.0:1775768249.010008:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a8a80 x1862022294292480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768274 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1775768249.010009:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:3.0:1775768249.010011:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:3.0:1775768249.010013:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:3.0:1775768249.010016:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768249.010017:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1775768249.010017:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775768249.010018:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000040:1.0:1775768249.010020:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a8a80 x1862022294292480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768274 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768249.010020:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775768249.010023:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:3.0:1775768249.010025:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775768249.010028:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768249.010028:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768249.010029:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775768249.010031:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775768249.010033:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768249.010034:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775768249.010034:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775768249.010036:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:3.0:1775768249.010038:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768249.010039:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768249.010039:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775768249.010042:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aa680. 00000400:00000200:1.0:1775768249.010043:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000040:3.0:1775768249.010044:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00000100:00000001:1.0:1775768249.010046:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4300175165 02000000:00000001:3.0:1775768249.010047:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775768249.010048:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768249.010049:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775768249.010050:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000010:1.0:1775768249.010051:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba3c0. 02000000:00000001:3.0:1775768249.010052:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775768249.010055:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0c00. 00000100:00000001:1.0:1775768249.010058:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1775768249.010058:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768249.010062:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aa680 x1862022294292864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775768249.010065:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4300175165 00000100:00000010:1.0:1775768249.010068:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba6c0. 00000100:00000001:3.0:1775768249.010069:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775768249.010070:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768249.010071:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775768249.010074:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1775768249.010074:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006cb1597d] to pc [ptlrpcd_rcv+-1] req@ffff9942099aa680 x1862022294292864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768249.010077:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768249.010080:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1775768249.010081:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768249.010083:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768249/real 1775768249] req@ffff9942099a8a80 x1862022294292480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768249.010083:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768249.010087:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775768249.010088:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775768249.010090:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775768249.010092:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000400:00000001:1.0:1775768249.010093:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:3.0:1775768249.010094:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5501 00000400:00000001:1.0:1775768249.010095:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768249.010097:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c738. 00000400:00000200:3.0:1775768249.010098:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768249.010100:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf65d8 00000400:00000200:3.0:1775768249.010100:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000400:00000010:1.0:1775768249.010102:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf65d8. 00000100:00080000:3.0:1775768249.010102:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1775768249.010105:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:3.0:1775768249.010105:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768249.010107:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:1.0:1775768249.010108:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a8a80 x1862022294292480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775768249.010109:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768249.010111:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775768249.010114:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775768249.010115:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:3.0:1775768249.010116:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000200:1.0:1775768249.010118:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a8a80 x1862022294292480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1775768249.010118:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:3.0:1775768249.010120:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:3.0:1775768249.010122:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:3.0:1775768249.010124:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768249.010125:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775768249.010126:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775768249.010127:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775768249.010128:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768249.010129:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775768249.010129:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:1.0:1775768249.010132:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1775768249.010132:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:3.0:1775768249.010134:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00100000:1.0:1775768249.010135:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a8a80 x1862022294292480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768249.010136:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768249.010138:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775768249.010140:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768249.010142:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1775768249.010142:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768249.010144:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775768249.010145:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000100:00000040:1.0:1775768249.010147:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a8a80 x1862022294292480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768274 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1775768249.010147:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768249.010149:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775768249.010152:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aad80. 00000020:00000040:3.0:1775768249.010171:0:13471:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00000100:00000001:1.0:1775768249.010174:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:3.0:1775768249.010174:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775768249.010176:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768249.010177:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768249.010177:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1775768249.010179:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:3.0:1775768249.010179:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1775768249.010180:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1775768249.010180:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768249.010183:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 02000000:00000010:3.0:1775768249.010184:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b1000. 00000100:00000001:1.0:1775768249.010185:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775768249.010187:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768249.010188:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1775768249.010192:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aad80 x1862022294292992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768249.010193:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768249.010196:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1775768249.010199:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:1.0:1775768249.010200:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a8a80 x1862022294292480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768274 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768249.010201:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768249.010204:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000045ccf08] to pc [ptlrpcd_rcv+-1] req@ffff9942099aad80 x1862022294292992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768249.010209:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294292480:192.168.202.156@tcp:250:kworker.0 00000100:00000001:3.0:1775768249.010212:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768249.010216:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775768249.010217:0:13471:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5511) 00000100:00000001:1.0:1775768249.010218:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768249.010221:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a8a80 x1862022294292480/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768274 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768249.010228:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768249.010231:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768249.010234:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543d800. 02000000:00000001:1.0:1775768249.010237:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768249.010239:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768249.010241:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:1.0:1775768249.010243:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768249.010246:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b2c00. 02000000:00000001:1.0:1775768249.010250:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768249.010252:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768249.010254:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a8a80. 00000100:00000001:1.0:1775768249.010257:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768249.010258:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768249.010260:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768249.010266:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768249.010267:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768249.010268:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768249.010269:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768249.010270:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768249.010273:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099aa300 x1862022294292608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768249.010277:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768249.010278:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768249.010279:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768249.010281:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768249.010282:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768249.010284:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099aa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294292608:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768249.010286:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768249.010287:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768249.010288:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768249.010290:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768249.010292:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543d800. 02000000:00000001:1.0:1775768249.010293:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768249.010296:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768249.010297:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c738. 00000400:00000010:1.0:1775768249.010299:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf65d8. 00000100:00000200:1.0:1775768249.010302:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294292608, portal 10 00000100:00000001:1.0:1775768249.010303:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768249.010304:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640861440 : -112966068690176 : ffff9942099aa300) 00000100:00000040:1.0:1775768249.010308:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099aa300 x1862022294292608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768274 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768249.010312:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768249.010314:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768249.010315:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000100:00000200:1.0:1775768249.010317:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294292608, offset 0 00000400:00000200:1.0:1775768249.010319:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768249.010321:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768249.010324:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768249.010326:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768249.010327:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768249.010329:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768249.010330:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768249.010332:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225459:225459:256:4294967295] 192.168.202.156@tcp LPNI seq info [225455:225455:8:4294967295] 00000400:00000200:1.0:1775768249.010335:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768249.010339:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768249.010341:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768249.010343:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541300. 00000800:00000200:1.0:1775768249.010345:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768249.010346:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775768249.010349:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768249.010351:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1775768249.010353:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768249.010354:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541300. 00000400:00000200:1.0:1775768249.010356:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768249.010359:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768249.010360:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000400:00000010:1.0:1775768249.010361:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000100:00000001:1.0:1775768249.010363:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768249.010365:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099aa300 x1862022294292608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768274 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768249.010370:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768249.010378:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099aa300 x1862022294292608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768274 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768249.010382:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768249.010383:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768249.010385:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768249.010388:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:1.0:1775768249.010390:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768249.010392:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300173843 00000100:00000001:1.0:1775768249.010393:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300175165 00000100:00000001:1.0:1775768249.010395:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300173843 00000100:00000001:1.0:1775768249.010396:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300175165 00000100:00000001:1.0:1775768249.010397:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768249.010399:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768249.010400:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768249.010401:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768249/real 1775768249] req@ffff9942099aa300 x1862022294292608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768249.010406:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768249.010407:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768249.010408:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c738. 00000400:00000200:1.0:1775768249.010410:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf65d8 00000400:00000010:1.0:1775768249.010411:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf65d8. 00000100:00000001:1.0:1775768249.010412:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768249.010414:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099aa300 x1862022294292608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768249.010419:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099aa300 x1862022294292608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768249.010423:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768249.010423:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768249.010424:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768249.010425:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768249.010427:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099aa300 x1862022294292608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768249.010435:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768249.010437:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099aa300 x1862022294292608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768274 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768249.010440:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768249.010442:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768249.010442:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768249.010443:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768249.010444:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768249.010446:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768249.010448:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768249.010450:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768249.010451:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768249.010453:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099aa300 x1862022294292608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768274 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768249.010457:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099aa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294292608:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768249.010459:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768249.010461:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ab800 x1862022294292736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768249.010468:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768249.010469:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768249.010470:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768249.010471:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768249.010473:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768249.010474:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294292736:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768249.010475:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768249.010476:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768249.010477:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768249.010478:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768249.010480:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543f800. 02000000:00000001:1.0:1775768249.010481:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768249.010483:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768249.010494:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c738. 00000400:00000010:1.0:1775768249.010497:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf65d8. 00000100:00000200:1.0:1775768249.010500:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294292736, portal 10 00000100:00000001:1.0:1775768249.010502:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768249.010503:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640866816 : -112966068684800 : ffff9942099ab800) 00000100:00000040:1.0:1775768249.010509:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ab800 x1862022294292736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768274 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768249.010515:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768249.010518:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768249.010520:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000100:00000200:1.0:1775768249.010522:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294292736, offset 0 00000400:00000200:1.0:1775768249.010526:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768249.010536:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768249.010539:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768249.010542:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768249.010544:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768249.010546:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768249.010548:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768249.010552:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225460:225460:256:4294967295] 192.168.202.156@tcp LPNI seq info [225456:225456:8:4294967295] 00000400:00000200:1.0:1775768249.010556:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768249.010562:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768249.010566:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768249.010569:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541300. 00000800:00000200:1.0:1775768249.010573:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768249.010575:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775768249.010579:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768249.010582:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1775768249.010585:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768249.010587:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541300. 00000400:00000200:1.0:1775768249.010589:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768249.010592:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768249.010595:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000400:00000010:1.0:1775768249.010596:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000100:00000001:1.0:1775768249.010598:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768249.010602:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ab800 x1862022294292736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768274 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768249.010609:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768249.010611:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ab800 x1862022294292736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768274 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768249.010618:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768249.010619:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768249.010622:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768249.010625:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:1.0:1775768249.010628:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768249.010629:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300173835 00000100:00000001:1.0:1775768249.010631:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300175165 00000100:00000001:1.0:1775768249.010633:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300173835 00000100:00000001:1.0:1775768249.010635:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300175165 00000100:00000001:1.0:1775768249.010638:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768249.010640:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768249.010641:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768249.010644:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768249/real 1775768249] req@ffff9942099ab800 x1862022294292736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768249.010652:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768249.010653:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768249.010654:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c738. 00000400:00000200:1.0:1775768249.010657:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf65d8 00000400:00000010:1.0:1775768249.010658:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf65d8. 00000100:00000001:1.0:1775768249.010660:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768249.010663:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ab800 x1862022294292736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768249.010670:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ab800 x1862022294292736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768249.010675:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768249.010677:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768249.010679:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768249.010680:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768249.010683:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ab800 x1862022294292736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768249.010689:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768249.010693:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ab800 x1862022294292736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768274 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768249.010699:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768249.010701:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768249.010703:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768249.010704:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768249.010706:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768249.010709:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768249.010711:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768249.010715:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768249.010717:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768249.010721:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ab800 x1862022294292736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768274 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768249.010728:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294292736:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768249.010732:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768249.010734:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099aa680 x1862022294292864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768249.010738:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768249.010739:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768249.010740:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768249.010742:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768249.010743:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768249.010744:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099aa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294292864:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768249.010746:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768249.010747:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768249.010748:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768249.010749:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768249.010752:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543fc00. 02000000:00000001:1.0:1775768249.010753:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768249.010756:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768249.010757:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c738. 00000400:00000010:1.0:1775768249.010760:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf65d8. 00000100:00000200:1.0:1775768249.010762:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294292864, portal 4 00000100:00000001:1.0:1775768249.010763:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768249.010764:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640862336 : -112966068689280 : ffff9942099aa680) 00000100:00000040:1.0:1775768249.010767:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099aa680 x1862022294292864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768274 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768249.010771:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768249.010773:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768249.010774:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000100:00000200:1.0:1775768249.010776:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294292864, offset 0 00000400:00000200:1.0:1775768249.010778:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768249.010781:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768249.010783:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768249.010785:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768249.010786:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768249.010788:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768249.010789:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768249.010792:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225461:225461:256:4294967295] 192.168.202.156@tcp LPNI seq info [225457:225457:8:4294967295] 00000400:00000200:1.0:1775768249.010794:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768249.010797:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768249.010800:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768249.010802:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541300. 00000800:00000200:1.0:1775768249.010804:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768249.010806:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775768249.010808:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768249.010810:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1775768249.010813:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768249.010814:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541300. 00000400:00000200:1.0:1775768249.010816:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768249.010818:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768249.010819:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000400:00000010:1.0:1775768249.010820:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000100:00000001:1.0:1775768249.010822:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768249.010824:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099aa680 x1862022294292864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768274 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768249.010829:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768249.010832:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099aa680 x1862022294292864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768274 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768249.010838:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768249.010840:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768249.010843:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768249.010846:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:1.0:1775768249.010849:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768249.010851:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300173858 00000100:00000001:1.0:1775768249.010853:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300175165 00000100:00000001:1.0:1775768249.010855:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300173858 00000100:00000001:1.0:1775768249.010857:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300175165 00000100:00000001:1.0:1775768249.010859:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768249.010861:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768249.010863:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768249.010866:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768249/real 1775768249] req@ffff9942099aa680 x1862022294292864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768249.010873:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768249.010874:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768249.010876:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c738. 00000400:00000200:1.0:1775768249.010878:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf65d8 00000400:00000010:1.0:1775768249.010880:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf65d8. 00000100:00000001:1.0:1775768249.010881:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768249.010884:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099aa680 x1862022294292864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768249.010892:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099aa680 x1862022294292864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768249.010899:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768249.010900:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768249.010902:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768249.010903:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768249.010906:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099aa680 x1862022294292864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768249.010913:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768249.010916:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099aa680 x1862022294292864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768274 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768249.010923:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768249.010925:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768249.010926:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768249.010928:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768249.010929:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768249.010934:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768249.010936:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768249.010938:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768249.010940:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768249.010943:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099aa680 x1862022294292864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768274 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768249.010950:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099aa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294292864:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768249.010954:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768249.010957:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099aad80 x1862022294292992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768249.010963:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768249.010964:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768249.010966:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768249.010969:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768249.010971:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768249.010973:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099aad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294292992:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768249.010977:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768249.010978:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768249.010980:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768249.010981:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768249.010985:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543e400. 02000000:00000001:1.0:1775768249.010986:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768249.010990:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768249.010993:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c738. 00000400:00000010:1.0:1775768249.010997:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf65d8. 00000100:00000200:1.0:1775768249.011000:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294292992, portal 4 00000100:00000001:1.0:1775768249.011002:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768249.011005:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640864128 : -112966068687488 : ffff9942099aad80) 00000100:00000040:1.0:1775768249.011011:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099aad80 x1862022294292992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768274 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768249.011018:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768249.011020:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768249.011022:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000100:00000200:1.0:1775768249.011024:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294292992, offset 0 00000400:00000200:1.0:1775768249.011028:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768249.011032:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768249.011034:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768249.011036:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768249.011037:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768249.011039:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768249.011040:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768249.011042:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225462:225462:256:4294967295] 192.168.202.156@tcp LPNI seq info [225458:225458:8:4294967295] 00000400:00000200:1.0:1775768249.011044:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768249.011048:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768249.011051:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768249.011053:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541300. 00000800:00000200:1.0:1775768249.011055:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768249.011057:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775768249.011060:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768249.011062:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1775768249.011064:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768249.011065:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541300. 00000400:00000200:1.0:1775768249.011067:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768249.011069:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768249.011071:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000400:00000010:1.0:1775768249.011072:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000100:00000001:1.0:1775768249.011074:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768249.011076:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099aad80 x1862022294292992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768274 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768249.011081:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768249.011083:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099aad80 x1862022294292992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768274 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768249.011086:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768249.011087:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768249.011089:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768249.011091:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:1.0:1775768249.011092:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768249.011094:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300173861 00000100:00000001:1.0:1775768249.011095:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300175165 00000100:00000001:1.0:1775768249.011096:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300173861 00000100:00000001:1.0:1775768249.011098:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300175165 00000100:00000001:1.0:1775768249.011099:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768249.011100:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768249.011101:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768249.011103:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768249/real 1775768249] req@ffff9942099aad80 x1862022294292992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768249.011107:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768249.011108:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768249.011109:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c738. 00000400:00000200:1.0:1775768249.011111:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf65d8 00000400:00000010:1.0:1775768249.011111:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf65d8. 00000100:00000001:1.0:1775768249.011112:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768249.011114:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099aad80 x1862022294292992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768249.011119:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099aad80 x1862022294292992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768249.011126:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768249.011127:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768249.011128:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768249.011129:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768249.011131:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099aad80 x1862022294292992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768249.011134:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768249.011136:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099aad80 x1862022294292992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768274 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768249.011140:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768249.011141:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768249.011142:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768249.011142:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768249.011143:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768249.011146:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768249.011147:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768249.011149:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768249.011150:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768249.011152:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099aad80 x1862022294292992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768274 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768249.011172:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099aad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294292992:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768249.011174:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768249.011175:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768249.011177:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099aa300 x1862022294292608/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768274 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768249.011181:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768249.011182:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768249.011184:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543d800. 02000000:00000001:1.0:1775768249.011185:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768249.011186:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768249.011187:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:1.0:1775768249.011189:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768249.011190:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b2800. 02000000:00000001:1.0:1775768249.011191:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768249.011192:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768249.011193:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099aa300. 00000100:00000001:1.0:1775768249.011196:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768249.011197:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768249.011198:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768249.011201:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ab800 x1862022294292736/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768274 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768249.011208:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768249.011209:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768249.011210:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543f800. 02000000:00000001:1.0:1775768249.011213:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768249.011214:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768249.011215:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:1.0:1775768249.011217:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768249.011218:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b0000. 02000000:00000001:1.0:1775768249.011221:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768249.011222:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768249.011223:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ab800. 00000100:00000001:1.0:1775768249.011226:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768249.011227:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768249.011229:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768249.011232:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099aa680 x1862022294292864/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768274 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768249.011238:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768249.011239:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768249.011241:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543fc00. 02000000:00000001:1.0:1775768249.011243:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768249.011244:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768249.011245:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:1.0:1775768249.011247:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768249.011249:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b0c00. 02000000:00000001:1.0:1775768249.011251:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768249.011252:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768249.011253:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099aa680. 00000100:00000001:1.0:1775768249.011255:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768249.011256:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768249.011258:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768249.011261:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099aad80 x1862022294292992/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768274 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768249.011267:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768249.011268:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768249.011269:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543e400. 02000000:00000001:1.0:1775768249.011272:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768249.011273:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768249.011274:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:1.0:1775768249.011276:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768249.011277:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b1000. 02000000:00000001:1.0:1775768249.011279:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768249.011280:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768249.011281:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099aad80. 00000100:00000001:1.0:1775768249.011283:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768249.011284:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768249.011286:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768249.011290:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768249.011291:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768249.011293:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768249.011294:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768249.073308:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768249.073313:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768249.073321:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768249.073323:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768249.073326:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768249.073327:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768249.073329:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768249.073330:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768249.073333:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768249.073334:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768249.073411:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768249.073415:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768249.073419:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768249.137309:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768249.137318:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768249.137320:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768249.137326:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768249.137330:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768249.137335:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768249.137338:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768249.137341:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768249.137345:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768250.033289:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768250.033297:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768250.033297:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768250.033298:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768250.033302:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768250.033303:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768250.033305:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768250.033306:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768250.033309:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768250.033311:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:1.0:1775768250.033314:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775768250.033315:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:0.0:1775768250.033317:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768250.033320:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768250.033321:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768250.033321:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768250.033323:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768250.033325:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768250.033326:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768250.033331:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768250.033333:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768250.033347:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768250.033349:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768250.033352:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768250.033353:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768250.033356:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768250.033359:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768250.097249:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768250.097257:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768250.097265:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768250.097267:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768250.097271:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768250.097272:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768250.097276:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768250.097277:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768250.097281:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768250.097282:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768250.097348:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768250.097352:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768250.097356:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768250.161246:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768250.161253:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768250.161255:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768250.161260:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768250.161263:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768250.161267:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768250.161269:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768250.161271:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768250.161273:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768250.353249:0:15036:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4300175165, at 4300175168 00000100:00000040:1.0:1775768250.353249:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4300175165, at 4300175168 00000100:00000010:1.0:1775768250.353266:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cba6c0. 00000100:00000010:0.0:1775768250.353272:0:15036:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cba3c0. 00000100:00000001:1.0:1775768251.057265:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768251.057265:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768251.057271:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768251.057272:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768251.057274:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768251.057275:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768251.057282:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775768251.057282:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768251.057288:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768251.057288:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768251.057293:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768251.057293:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768251.057296:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768251.057299:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768251.057301:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768251.057302:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768251.057304:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768251.057306:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768251.057393:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768251.057397:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768251.057398:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768251.057401:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775768251.057403:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768251.057404:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768251.057405:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768251.057407:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768251.057408:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768251.121291:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768251.121297:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768251.121302:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768251.121303:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768251.121306:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768251.121307:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768251.121309:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768251.121310:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768251.121313:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768251.121314:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768251.121394:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768251.121399:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768251.121403:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768251.185288:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768251.185295:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768251.185297:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768251.185302:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768251.185315:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768251.185320:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768251.185323:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768251.185325:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768251.185327:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768252.081299:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768252.081302:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768252.081311:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768252.081312:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768252.081317:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768252.081317:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768252.081326:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:1.0:1775768252.081327:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768252.081337:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768252.081338:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768252.081346:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768252.081346:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768252.081354:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768252.081357:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768252.081361:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768252.081362:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768252.081366:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768252.081369:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768252.081523:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768252.081526:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768252.081528:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768252.081530:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768252.081532:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768252.081534:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768252.081535:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768252.081536:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768252.081537:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768252.145253:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768252.145260:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768252.145262:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768252.145266:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768252.145270:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768252.145273:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768252.145274:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768252.145277:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768252.145280:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768252.145281:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768252.145283:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768252.145284:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768252.145286:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768252.145287:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768252.145289:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768252.145290:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768252.145361:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768252.145365:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768252.145368:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768252.209262:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768252.209268:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768252.209270:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768252.209275:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768252.209278:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768252.209282:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768252.209284:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768252.209286:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768252.209288:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768253.105271:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768253.105277:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768253.105279:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768253.105284:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768253.105287:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768253.105291:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768253.105293:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768253.105295:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768253.105298:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768253.106536:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768253.106539:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768253.106541:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768253.106544:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775768253.106546:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768253.106548:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768253.106550:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768253.106559:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768253.106561:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768253.107522:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768253.107525:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768253.107527:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768253.107529:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768253.107531:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768253.107533:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768253.107535:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768253.107536:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768253.107539:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768253.169349:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768253.169355:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768253.169361:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768253.169362:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768253.169365:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768253.169366:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768253.169370:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768253.169371:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768253.169374:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768253.169375:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768253.169429:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768253.169433:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768253.169437:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768253.233253:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768253.233260:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768253.233262:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768253.233267:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768253.233272:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768253.233276:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768253.233278:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768253.233280:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768253.233282:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768254.129244:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775768254.129247:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768254.129252:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:3.0:1775768254.129254:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775768254.129255:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768254.129258:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:0.0:1775768254.129260:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00080000:3.0:1775768254.129262:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768254.129264:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768254.129269:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768254.129269:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:0.0:1775768254.129274:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768254.129277:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775768254.129278:0:13471:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5506 00000100:00000001:0.0:1775768254.129282:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775768254.129291:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775768254.129298:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775768254.129301:0:13471:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775768254.129305:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768254.129308:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775768254.129311:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768254.129314:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775768254.129318:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775768254.129321:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:3.0:1775768254.129324:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775768254.129326:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:3.0:1775768254.129329:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:3.0:1775768254.129332:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:3.0:1775768254.129346:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768254.129349:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775768254.129352:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768254.129354:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775768254.129357:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:3.0:1775768254.129361:0:13471:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775768254.129364:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768254.129365:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768254.129369:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768254.129370:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768254.129370:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:2.0:1775768254.129373:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 02000000:00000001:3.0:1775768254.129374:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768254.129375:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768254.129378:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1775768254.129380:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768254.129381:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768254.129384:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768254.129384:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775768254.129386:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768254.129388:0:13471:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775768254.129398:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aad80. 00000100:00000001:1.0:1775768254.129400:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768254.129404:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000020:00000040:3.0:1775768254.129404:0:13471:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 00000100:00000001:1.0:1775768254.129407:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768254.129411:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 02000000:00000001:3.0:1775768254.129411:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1775768254.129415:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1775768254.129415:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768254.129417:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768254.129420:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768254.129422:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768254.129424:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768254.129429:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775768254.129432:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775768254.129450:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775768254.129460:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2000. 00000100:00000001:3.0:1775768254.129466:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768254.129475:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aad80 x1862022294293120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775768254.129485:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775768254.129490:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768254.129496:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000045ccf08] to pc [ptlrpcd_rcv+-1] req@ffff9942099aad80 x1862022294293120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768254.129517:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768254.129520:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768254.129523:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775768254.129524:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775768254.129526:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775768254.129529:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775768254.129531:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5506 00000400:00000200:3.0:1775768254.129535:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775768254.129539:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:1.0:1775768254.129549:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1775768254.129550:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1775768254.129552:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768254.129556:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775768254.129556:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768254.129560:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768254.129562:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099aad80 x1862022294293120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775768254.129565:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768254.129571:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:1.0:1775768254.129575:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000001:3.0:1775768254.129576:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:1.0:1775768254.129577:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775768254.129579:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 02000000:00000001:1.0:1775768254.129581:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:3.0:1775768254.129582:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000001:1.0:1775768254.129584:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:3.0:1775768254.129585:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 02000000:00000001:1.0:1775768254.129586:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768254.129591:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00100000:1.0:1775768254.129592:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099aad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294293120:192.168.202.156@tcp:250:kworker.0 00000020:00000001:3.0:1775768254.129596:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000001:1.0:1775768254.129597:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768254.129602:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:3.0:1775768254.129603:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 02000000:00000001:1.0:1775768254.129604:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768254.129608:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1775768254.129609:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775768254.129612:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000010:1.0:1775768254.129613:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543e400. 02000000:00000001:1.0:1775768254.129617:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768254.129617:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775768254.129622:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000400:00000200:1.0:1775768254.129625:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768254.129629:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c738. 00000100:00080000:3.0:1775768254.129629:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775768254.129634:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775768254.129636:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf65d8. 02000000:00000001:3.0:1775768254.129637:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775768254.129644:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000200:1.0:1775768254.129645:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294293120, portal 25 00000100:00000001:1.0:1775768254.129649:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1775768254.129649:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768254.129651:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640864128 : -112966068687488 : ffff9942099aad80) 00000100:00000001:3.0:1775768254.129653:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000040:1.0:1775768254.129657:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099aad80 x1862022294293120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768279 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768254.129662:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768254.129664:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768254.129666:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000008:00100000:3.0:1775768254.129666:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00000200:1.0:1775768254.129672:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294293120, offset 0 00000008:00000001:3.0:1775768254.129672:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768254.129675:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768254.129681:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00000010:3.0:1775768254.129681:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aa680. 00000020:00000040:3.0:1775768254.129685:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:3.0:1775768254.129689:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775768254.129691:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768254.129693:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775768254.129694:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775768254.129697:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775768254.129699:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768254.129702:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 02000000:00000010:3.0:1775768254.129705:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0400. 00000400:00000200:1.0:1775768254.129708:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1775768254.129710:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768254.129714:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000100:00100000:3.0:1775768254.129719:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aa680 x1862022294293248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775768254.129720:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768254.129726:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768254.129733:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225463:225463:256:4294967295] 192.168.202.156@tcp LPNI seq info [225459:225459:8:4294967295] 00000100:00000001:3.0:1775768254.129736:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1775768254.129739:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775768254.129741:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768254.129749:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006cb1597d] to pc [ptlrpcd_rcv+-1] req@ffff9942099aa680 x1862022294293248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775768254.129751:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768254.129760:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768254.129764:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541300. 00000800:00000200:1.0:1775768254.129770:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768254.129774:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000100:00000001:3.0:1775768254.129775:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768254.129781:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:1.1:1775768254.129782:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768254.129786:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000100:00080000:3.0:1775768254.129788:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000100:1.0:1775768254.129792:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000100:00000001:3.0:1775768254.129792:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775768254.129796:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000010:1.0:1775768254.129797:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541300. 00000100:00000001:3.0:1775768254.129802:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:1.0:1775768254.129803:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00080000:3.0:1775768254.129806:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5506 00000400:00000200:1.0:1775768254.129810:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768254.129815:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000400:00000200:3.0:1775768254.129816:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768254.129817:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000100:00000001:1.0:1775768254.129821:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000400:00000200:3.0:1775768254.129822:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000200:1.0:1775768254.129827:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099aad80 x1862022294293120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768279 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768254.129827:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775768254.129834:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768254.129840:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775768254.129844:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768254.129850:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:1.0:1775768254.129851:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000020:00000001:3.0:1775768254.129854:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000040:1.0:1775768254.129856:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099aad80 x1862022294293120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768279 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1775768254.129857:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000100:00000001:1.0:1775768254.129864:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768254.129865:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:3.0:1775768254.129868:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:1.0:1775768254.129869:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775768254.129872:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000400:00000200:1.0:1775768254.129875:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:3.0:1775768254.129876:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000001:1.0:1775768254.129878:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4300180285 00000020:00000001:3.0:1775768254.129881:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000010:1.0:1775768254.129884:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba6c0. 00000100:00000040:3.0:1775768254.129887:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768254.129890:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775768254.129892:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000040:3.0:1775768254.129893:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768254.129897:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:1.0:1775768254.129899:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4300180285 00000020:00000040:3.0:1775768254.129900:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00000010:1.0:1775768254.129906:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cbaba0. 00000100:00080000:3.0:1775768254.129906:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775768254.129909:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768254.129910:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775768254.129913:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775768254.129918:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768254.129920:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1775768254.129921:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775768254.129925:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1775768254.129926:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00000001:3.0:1775768254.129929:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768254.129931:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768254.129932:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768254.129935:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000010:3.0:1775768254.129938:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ab800. 00000100:00100000:1.0:1775768254.129942:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768254/real 1775768254] req@ffff9942099aad80 x1862022294293120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1775768254.129942:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:3.0:1775768254.129946:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775768254.129948:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768254.129950:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775768254.129951:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775768254.129954:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775768254.129958:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b3400. 00000400:00000001:1.0:1775768254.129961:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1775768254.129962:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1775768254.129963:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768254.129966:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c738. 00000400:00000200:1.0:1775768254.129969:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf65d8 00000100:00100000:3.0:1775768254.129970:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ab800 x1862022294293376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:1.0:1775768254.129972:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf65d8. 00000100:00000001:1.0:1775768254.129975:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768254.129981:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099aad80 x1862022294293120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768254.129989:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775768254.129993:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775768254.129995:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099aad80 x1862022294293120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775768254.130001:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000089df80c0] to pc [ptlrpcd_rcv+-1] req@ffff9942099ab800 x1862022294293376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768254.130002:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768254.130004:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768254.130009:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768254.130011:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768254.130017:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099aad80 x1862022294293120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768254.130019:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768254.130024:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775768254.130025:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775768254.130030:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000040:1.0:1775768254.130031:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099aad80 x1862022294293120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768279 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768254.130034:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775768254.130037:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768254.130041:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775768254.130042:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775768254.130044:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768254.130046:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:3.0:1775768254.130047:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5506 00000100:00000001:1.0:1775768254.130048:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768254.130055:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768254.130056:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768254.130058:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768254.130061:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768254.130062:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000400:00000200:3.0:1775768254.130064:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000040:1.0:1775768254.130067:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099aad80 x1862022294293120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768279 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1775768254.130071:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775768254.130077:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775768254.130081:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099aad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294293120:192.168.202.156@tcp:250:kworker.0 00000100:00000040:3.0:1775768254.130082:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768254.130087:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775768254.130088:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768254.130091:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775768254.130093:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000040:1.0:1775768254.130095:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099aad80 x1862022294293120/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768279 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768254.130097:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775768254.130102:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775768254.130104:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:3.0:1775768254.130108:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:1.0:1775768254.130109:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 00000020:00000040:3.0:1775768254.130112:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 02000000:00000001:1.0:1775768254.130115:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000001:3.0:1775768254.130115:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 02000000:00000010:1.0:1775768254.130118:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543e400. 00000020:00000001:3.0:1775768254.130119:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 02000000:00000001:1.0:1775768254.130121:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768254.130124:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768254.130126:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000100:00000040:3.0:1775768254.130127:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000001:1.0:1775768254.130130:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1775768254.130131:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000010:1.0:1775768254.130133:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b2000. 00000100:00000040:3.0:1775768254.130135:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000001:1.0:1775768254.130139:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:3.0:1775768254.130139:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775768254.130142:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 02000000:00000001:1.0:1775768254.130143:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768254.130146:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099aad80. 00000100:00080000:3.0:1775768254.130148:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775768254.130152:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775768254.130152:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768254.130173:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768254.130176:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1775768254.130182:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1775768254.130183:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768254.130185:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768254.130186:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768254.130189:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1775768254.130191:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768254.130192:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768254.130195:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1775768254.130195:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775768254.130199:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000100:00000040:1.0:1775768254.130202:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099aa680 x1862022294293248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1775768254.130203:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768254.130207:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775768254.130213:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099aa300. 00000100:00000001:1.0:1775768254.130215:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000040:3.0:1775768254.130217:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00000100:00000001:1.0:1775768254.130218:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768254.130222:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775768254.130223:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768254.130225:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775768254.130226:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768254.130231:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775768254.130232:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768254.130233:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775768254.130237:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768254.130239:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099aa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294293248:192.168.202.156@tcp:38:kworker.0 02000000:00000010:3.0:1775768254.130242:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2000. 00000100:00000001:1.0:1775768254.130244:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768254.130248:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1775768254.130248:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768254.130250:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768254.130253:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:3.0:1775768254.130257:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099aa300 x1862022294293504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000010:1.0:1775768254.130259:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543e400. 02000000:00000001:1.0:1775768254.130263:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768254.130272:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775768254.130272:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775768254.130277:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775768254.130278:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c738. 00000400:00000010:1.0:1775768254.130284:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf65d8. 00000100:00000040:3.0:1775768254.130285:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f33bdf15] to pc [ptlrpcd_rcv+-1] req@ffff9942099aa300 x1862022294293504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768254.130294:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768254.130298:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:1.0:1775768254.130299:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294293248, portal 10 00000100:00000001:1.0:1775768254.130302:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1775768254.130304:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775768254.130305:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640862336 : -112966068689280 : ffff9942099aa680) 00000100:00000001:3.0:1775768254.130308:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775768254.130310:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775768254.130313:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000040:1.0:1775768254.130316:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099aa680 x1862022294293248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768279 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768254.130317:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5506 00000400:00000200:3.0:1775768254.130326:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768254.130333:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:3.0:1775768254.130333:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775768254.130338:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775768254.130341:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000100:00000040:3.0:1775768254.130345:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000010:1.0:1775768254.130346:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000100:00000001:3.0:1775768254.130348:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:1.0:1775768254.130351:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294293248, offset 0 00000100:00000040:3.0:1775768254.130352:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768254.130357:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775768254.130361:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775768254.130363:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:3.0:1775768254.130365:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:1.0:1775768254.130367:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000020:00000040:3.0:1775768254.130367:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:3.0:1775768254.130371:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:3.0:1775768254.130375:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000400:00000200:1.0:1775768254.130376:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000040:3.0:1775768254.130381:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000200:1.0:1775768254.130384:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00000001:3.0:1775768254.130385:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1775768254.130390:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:3.0:1775768254.130391:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000200:1.0:1775768254.130394:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000100:00000001:3.0:1775768254.130395:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000200:1.0:1775768254.130398:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768254.130401:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000020:00000040:3.0:1775768254.130401:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:3.0:1775768254.130408:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000400:00000200:1.0:1775768254.130409:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225464:225464:256:4294967295] 192.168.202.156@tcp LPNI seq info [225460:225460:8:4294967295] 00000100:00000001:3.0:1775768254.130415:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768254.130418:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:3.0:1775768254.130419:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775768254.130423:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775768254.130428:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768254.130429:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000100:00000001:3.0:1775768254.130432:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000800:00000200:1.0:1775768254.130447:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000008:00100000:3.0:1775768254.130449:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000800:00000010:1.0:1775768254.130452:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541300. 00000008:00000001:3.0:1775768254.130454:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768254.130456:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775768254.130460:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8a80. 00000020:00000040:3.0:1775768254.130463:0:13471:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00000800:00000200:1.0:1775768254.130468:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 02000000:00000001:3.0:1775768254.130468:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775768254.130471:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775768254.130473:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000100:00000001:3.0:1775768254.130474:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775768254.130476:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775768254.130478:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775768254.130482:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768254.130484:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 02000000:00000010:3.0:1775768254.130485:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b1000. 00000800:00000100:1.0:1775768254.130489:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000100:00000001:3.0:1775768254.130490:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1775768254.130491:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541300. 00000400:00000200:1.0:1775768254.130495:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00100000:3.0:1775768254.130499:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8a80 x1862022294293632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775768254.130500:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768254.130505:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000400:00000010:1.0:1775768254.130507:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000100:00000001:1.0:1775768254.130511:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1775768254.130514:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000200:1.0:1775768254.130515:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099aa680 x1862022294293248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768279 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768254.130518:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768254.130526:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006c3265af] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8a80 x1862022294293632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768254.130530:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768254.130537:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099aa680 x1862022294293248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768279 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768254.130539:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768254.130546:0:13471:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5516) 00000100:00000001:1.0:1775768254.130555:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768254.130557:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768254.130562:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768254.130565:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:1.0:1775768254.130569:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768254.130571:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300178963 00000100:00000001:1.0:1775768254.130573:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300180285 00000100:00000001:1.0:1775768254.130576:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300178963 00000100:00000001:1.0:1775768254.130577:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300180285 00000100:00000001:1.0:1775768254.130579:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768254.130581:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768254.130583:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768254.130586:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768254/real 1775768254] req@ffff9942099aa680 x1862022294293248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768254.130591:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768254.130592:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768254.130594:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c738. 00000400:00000200:1.0:1775768254.130595:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf65d8 00000400:00000010:1.0:1775768254.130596:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf65d8. 00000100:00000001:1.0:1775768254.130598:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768254.130599:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099aa680 x1862022294293248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768254.130604:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099aa680 x1862022294293248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768254.130608:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768254.130608:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768254.130609:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768254.130610:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768254.130612:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099aa680 x1862022294293248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768254.130616:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768254.130618:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099aa680 x1862022294293248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768279 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768254.130622:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768254.130623:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768254.130624:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768254.130625:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768254.130626:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768254.130627:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768254.130629:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768254.130630:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768254.130632:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768254.130633:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099aa680 x1862022294293248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768279 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768254.130638:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099aa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294293248:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768254.130640:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768254.130642:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ab800 x1862022294293376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768254.130646:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768254.130647:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768254.130648:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768254.130649:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768254.130650:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768254.130651:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294293376:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768254.130653:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768254.130655:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768254.130656:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768254.130657:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768254.130659:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543fc00. 02000000:00000001:1.0:1775768254.130660:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768254.130662:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768254.130664:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c738. 00000400:00000010:1.0:1775768254.130666:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf65d8. 00000100:00000200:1.0:1775768254.130668:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294293376, portal 10 00000100:00000001:1.0:1775768254.130670:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768254.130671:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640866816 : -112966068684800 : ffff9942099ab800) 00000100:00000040:1.0:1775768254.130676:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ab800 x1862022294293376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768279 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768254.130682:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768254.130684:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768254.130686:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000100:00000200:1.0:1775768254.130688:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294293376, offset 0 00000400:00000200:1.0:1775768254.130692:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768254.130695:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768254.130699:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768254.130702:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768254.130704:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768254.130706:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768254.130709:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768254.130712:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225465:225465:256:4294967295] 192.168.202.156@tcp LPNI seq info [225461:225461:8:4294967295] 00000400:00000200:1.0:1775768254.130716:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768254.130721:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768254.130725:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768254.130728:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541300. 00000800:00000200:1.0:1775768254.130731:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768254.130733:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775768254.130735:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768254.130747:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000100:1.0:1775768254.130750:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768254.130752:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541300. 00000400:00000200:1.0:1775768254.130753:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768254.130756:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768254.130758:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000400:00000010:1.0:1775768254.130759:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000100:00000001:1.0:1775768254.130760:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768254.130763:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ab800 x1862022294293376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768279 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768254.130768:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768254.130769:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ab800 x1862022294293376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768279 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768254.130773:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768254.130774:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768254.130776:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768254.130778:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:1.0:1775768254.130780:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768254.130781:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300178955 00000100:00000001:1.0:1775768254.130783:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300180285 00000100:00000001:1.0:1775768254.130784:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300178955 00000100:00000001:1.0:1775768254.130785:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300180285 00000100:00000001:1.0:1775768254.130787:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768254.130788:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768254.130789:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768254.130791:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768254/real 1775768254] req@ffff9942099ab800 x1862022294293376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768254.130795:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768254.130796:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768254.130797:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c738. 00000400:00000200:1.0:1775768254.130798:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf65d8 00000400:00000010:1.0:1775768254.130799:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf65d8. 00000100:00000001:1.0:1775768254.130800:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768254.130802:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ab800 x1862022294293376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768254.130807:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ab800 x1862022294293376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768254.130810:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768254.130811:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768254.130812:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768254.130812:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768254.130814:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ab800 x1862022294293376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768254.130817:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768254.130819:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ab800 x1862022294293376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768279 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768254.130823:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768254.130824:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768254.130825:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768254.130825:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768254.130826:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768254.130828:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768254.130829:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768254.130831:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768254.130832:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768254.130834:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ab800 x1862022294293376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768279 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768254.130839:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294293376:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768254.130841:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768254.130842:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768254.130844:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099aa680 x1862022294293248/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768279 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768254.130848:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768254.130850:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768254.130851:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543e400. 02000000:00000001:1.0:1775768254.130853:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768254.130854:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768254.130856:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:1.0:1775768254.130858:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768254.130859:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b0400. 02000000:00000001:1.0:1775768254.130861:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768254.130863:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768254.130864:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099aa680. 00000100:00000001:1.0:1775768254.130866:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768254.130867:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768254.130868:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768254.130871:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ab800 x1862022294293376/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768279 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768254.130877:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768254.130878:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768254.130880:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543fc00. 02000000:00000001:1.0:1775768254.130882:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768254.130883:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768254.130884:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:1.0:1775768254.130886:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768254.130887:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b3400. 02000000:00000001:1.0:1775768254.130889:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768254.130890:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768254.130891:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ab800. 00000100:00000001:1.0:1775768254.130893:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768254.130894:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768254.130896:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768254.130899:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768254.130900:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768254.130902:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768254.130903:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768254.130905:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768254.130908:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099aa300 x1862022294293504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768254.130915:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768254.130916:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768254.130918:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768254.130920:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768254.130921:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768254.130924:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099aa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294293504:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768254.130927:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768254.130928:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768254.130930:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768254.130931:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768254.130934:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543fc00. 02000000:00000001:1.0:1775768254.130936:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768254.130939:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768254.130942:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c738. 00000400:00000010:1.0:1775768254.130944:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf65d8. 00000100:00000200:1.0:1775768254.130946:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294293504, portal 4 00000100:00000001:1.0:1775768254.130947:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768254.130948:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640861440 : -112966068690176 : ffff9942099aa300) 00000100:00000040:1.0:1775768254.130952:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099aa300 x1862022294293504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768279 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768254.130956:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768254.130957:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768254.130959:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000100:00000200:1.0:1775768254.130960:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294293504, offset 0 00000400:00000200:1.0:1775768254.130962:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768254.130965:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768254.130967:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768254.130968:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768254.130970:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768254.130972:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768254.130973:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768254.130975:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225466:225466:256:4294967295] 192.168.202.156@tcp LPNI seq info [225462:225462:8:4294967295] 00000400:00000200:1.0:1775768254.130977:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768254.130981:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768254.130983:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768254.130985:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541300. 00000800:00000200:1.0:1775768254.130987:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768254.130989:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775768254.130991:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768254.130993:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000100:1.0:1775768254.130995:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768254.130997:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541300. 00000400:00000200:1.0:1775768254.130999:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768254.131002:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768254.131005:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000400:00000010:1.0:1775768254.131006:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000100:00000001:1.0:1775768254.131008:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768254.131011:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099aa300 x1862022294293504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768279 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768254.131020:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768254.131022:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099aa300 x1862022294293504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768279 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768254.131027:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768254.131028:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768254.131031:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768254.131033:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:1.0:1775768254.131036:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768254.131038:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300178978 00000100:00000001:1.0:1775768254.131039:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300180285 00000100:00000001:1.0:1775768254.131041:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300178978 00000100:00000001:1.0:1775768254.131044:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300180285 00000100:00000001:1.0:1775768254.131046:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768254.131047:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768254.131049:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768254.131052:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768254/real 1775768254] req@ffff9942099aa300 x1862022294293504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768254.131059:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768254.131060:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768254.131061:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c738. 00000400:00000200:1.0:1775768254.131063:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf65d8 00000400:00000010:1.0:1775768254.131065:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf65d8. 00000100:00000001:1.0:1775768254.131067:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768254.131069:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099aa300 x1862022294293504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768254.131075:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099aa300 x1862022294293504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768254.131080:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768254.131081:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768254.131083:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768254.131084:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768254.131087:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099aa300 x1862022294293504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768254.131092:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768254.131095:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099aa300 x1862022294293504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768279 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768254.131100:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768254.131102:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768254.131103:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768254.131104:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768254.131106:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768254.131109:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768254.131111:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768254.131113:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768254.131115:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768254.131118:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099aa300 x1862022294293504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768279 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768254.131124:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099aa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294293504:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768254.131127:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768254.131130:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a8a80 x1862022294293632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768254.131135:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768254.131136:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768254.131138:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768254.131139:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768254.131141:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768254.131143:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294293632:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768254.131146:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768254.131147:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768254.131149:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768254.131150:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768254.131153:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543e400. 02000000:00000001:1.0:1775768254.131172:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768254.131176:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768254.131178:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c738. 00000400:00000010:1.0:1775768254.131180:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf65d8. 00000100:00000200:1.0:1775768254.131182:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294293632, portal 4 00000100:00000001:1.0:1775768254.131183:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768254.131184:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640855168 : -112966068696448 : ffff9942099a8a80) 00000100:00000040:1.0:1775768254.131189:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a8a80 x1862022294293632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768279 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768254.131193:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768254.131194:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768254.131195:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000100:00000200:1.0:1775768254.131196:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294293632, offset 0 00000400:00000200:1.0:1775768254.131198:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768254.131201:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768254.131203:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768254.131204:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768254.131206:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768254.131207:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768254.131209:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768254.131211:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225467:225467:256:4294967295] 192.168.202.156@tcp LPNI seq info [225463:225463:8:4294967295] 00000400:00000200:1.0:1775768254.131213:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768254.131216:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768254.131219:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768254.131221:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541300. 00000800:00000200:1.0:1775768254.131223:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768254.131225:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775768254.131227:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768254.131229:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 43 secs later) 00000800:00000100:1.0:1775768254.131231:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768254.131232:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541300. 00000400:00000200:1.0:1775768254.131234:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768254.131236:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768254.131238:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000400:00000010:1.0:1775768254.131239:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000100:00000001:1.0:1775768254.131240:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768254.131242:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a8a80 x1862022294293632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768279 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768254.131246:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768254.131248:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a8a80 x1862022294293632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768279 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768254.131255:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768254.131256:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768254.131258:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768254.131260:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:1.0:1775768254.131261:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768254.131262:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300178982 00000100:00000001:1.0:1775768254.131264:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300180285 00000100:00000001:1.0:1775768254.131265:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300178982 00000100:00000001:1.0:1775768254.131266:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300180285 00000100:00000001:1.0:1775768254.131267:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768254.131268:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768254.131269:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768254.131271:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768254/real 1775768254] req@ffff9942099a8a80 x1862022294293632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768254.131275:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768254.131276:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768254.131277:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c738. 00000400:00000200:1.0:1775768254.131278:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf65d8 00000400:00000010:1.0:1775768254.131279:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf65d8. 00000100:00000001:1.0:1775768254.131280:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768254.131282:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a8a80 x1862022294293632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768254.131286:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a8a80 x1862022294293632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768254.131289:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768254.131290:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768254.131291:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768254.131292:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768254.131293:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a8a80 x1862022294293632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768254.131298:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768254.131300:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a8a80 x1862022294293632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768279 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768254.131303:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768254.131304:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768254.131305:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768254.131306:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768254.131307:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768254.131309:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768254.131310:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768254.131312:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768254.131313:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768254.131315:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a8a80 x1862022294293632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768279 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768254.131319:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294293632:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768254.131321:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768254.131322:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768254.131323:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099aa300 x1862022294293504/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768279 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768254.131327:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768254.131328:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768254.131329:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543fc00. 02000000:00000001:1.0:1775768254.131330:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768254.131331:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768254.131331:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:1.0:1775768254.131332:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768254.131333:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b2000. 02000000:00000001:1.0:1775768254.131335:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768254.131336:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768254.131337:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099aa300. 00000100:00000001:1.0:1775768254.131338:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768254.131339:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768254.131340:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768254.131341:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a8a80 x1862022294293632/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768279 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768254.131345:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768254.131345:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768254.131346:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543e400. 02000000:00000001:1.0:1775768254.131347:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768254.131348:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768254.131348:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:1.0:1775768254.131349:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768254.131350:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b1000. 02000000:00000001:1.0:1775768254.131351:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768254.131352:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768254.131352:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a8a80. 00000100:00000001:1.0:1775768254.131353:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768254.131354:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768254.131355:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768254.131357:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768254.131358:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768254.131359:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768254.131359:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768254.193265:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768254.193271:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768254.193276:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768254.193278:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768254.193282:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768254.193284:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768254.193287:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768254.193289:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768254.193292:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768254.193293:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768254.193350:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768254.193354:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768254.193357:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768254.257264:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768254.257270:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768254.257271:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768254.257275:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768254.257278:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768254.257281:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768254.257282:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768254.257284:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768254.257286:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768255.153249:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768255.153249:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768255.153255:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768255.153256:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768255.153256:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768255.153260:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768255.153260:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768255.153261:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768255.153265:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768255.153267:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775768255.153267:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:1.0:1775768255.153269:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768255.153274:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768255.153274:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768255.153274:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768255.153279:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768255.153279:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768255.153280:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768255.153286:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768255.153289:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768255.153291:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768255.153292:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768255.153295:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768255.153298:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768255.153298:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768255.153301:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768255.153304:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768255.217251:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768255.217255:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768255.217258:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768255.217259:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768255.217262:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768255.217263:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768255.217265:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768255.217266:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768255.217268:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768255.217269:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768255.217323:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768255.217327:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768255.217331:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768255.281263:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768255.281269:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768255.281271:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768255.281276:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768255.281280:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768255.281283:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768255.281286:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768255.281289:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768255.281291:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768255.473695:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4300180285, at 4300180288 00000100:00000010:1.0:1775768255.473710:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cbaba0. 00000100:00000040:0.0:1775768255.474102:0:15036:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4300180285, at 4300180288 00000100:00000010:0.0:1775768255.474108:0:15036:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cba6c0. 00000100:00000001:0.0:1775768256.177256:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768256.177257:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768256.177264:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768256.177264:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768256.177268:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768256.177273:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775768256.177276:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768256.177277:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768256.177280:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768256.177281:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768256.177284:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768256.177285:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768256.177288:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768256.177289:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768256.177293:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768256.177294:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768256.177297:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768256.177300:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768256.177377:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768256.177380:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768256.177381:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768256.177384:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775768256.177386:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768256.177388:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768256.177389:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768256.177391:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768256.177393:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768256.241251:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768256.241255:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768256.241259:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768256.241261:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768256.241264:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768256.241269:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768256.241271:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768256.241272:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768256.241274:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768256.241275:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768256.241326:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768256.241330:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768256.241334:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768256.305233:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768256.305238:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768256.305239:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768256.305243:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768256.305246:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768256.305248:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768256.305250:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768256.305252:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768256.305253:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768257.201287:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768257.201287:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768257.201297:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768257.201301:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768257.201301:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768257.201305:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768257.201309:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:1.0:1775768257.201310:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768257.201315:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768257.201315:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768257.201324:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768257.201325:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768257.201331:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768257.201336:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768257.201340:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768257.201340:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768257.201343:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768257.201346:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768257.201373:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768257.201376:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768257.201378:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768257.201380:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768257.201384:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768257.201389:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768257.201391:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768257.201392:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768257.201394:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768257.265242:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768257.265246:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768257.265250:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768257.265252:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768257.265254:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768257.265256:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768257.265258:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768257.265260:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768257.265261:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768257.265263:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768257.265265:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768257.265266:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768257.265267:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768257.265267:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768257.265268:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768257.265269:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768257.265270:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768257.265271:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768257.265273:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768257.265274:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768257.265274:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768257.265275:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768257.265276:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768257.265276:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768257.265277:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768257.265278:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768257.265280:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768257.265281:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768257.265282:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768257.265283:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768257.265283:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768257.265284:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768257.265286:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768257.265287:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768257.265331:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768257.265334:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768257.265336:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768257.329319:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768257.329329:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768257.329331:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768257.329338:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768257.329342:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768257.329347:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768257.329350:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768257.329353:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768257.329356:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768258.225246:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768258.225246:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768258.225252:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768258.225253:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768258.225255:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768258.225256:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768258.225261:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:1.0:1775768258.225262:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775768258.225262:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768258.225266:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768258.225266:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768258.225269:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768258.225271:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768258.225273:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768258.225274:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768258.225276:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768258.225277:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768258.225280:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768258.225281:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768258.225283:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768258.225285:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768258.225286:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768258.225289:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768258.225290:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768258.225293:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768258.225296:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768258.225299:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768258.289256:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768258.289262:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768258.289268:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768258.289270:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768258.289274:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768258.289275:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768258.289278:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768258.289280:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768258.289282:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768258.289283:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768258.289347:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768258.289351:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768258.289354:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768258.353239:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768258.353245:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768258.353247:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768258.353253:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768258.353256:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768258.353260:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768258.353262:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768258.353274:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768258.353278:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768259.249228:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768259.249234:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768259.249236:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768259.249238:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:2.0:1775768259.249241:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768259.249242:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768259.249243:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768259.249244:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768259.249245:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1775768259.249248:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768259.249249:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768259.249249:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768259.249250:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768259.249252:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768259.249253:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768259.249253:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768259.249255:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768259.249258:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768259.249258:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768259.249259:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768259.249260:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768259.249265:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768259.249269:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768259.249271:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768259.249274:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768259.249276:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768259.249278:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768259.249343:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768259.249349:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775768259.249352:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768259.249355:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768259.249358:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768259.249364:0:13471:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5511 00000400:00000200:0.0:1775768259.249372:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768259.249382:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768259.249394:0:13471:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768259.249398:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768259.249401:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768259.249403:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768259.249405:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768259.249409:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768259.249411:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:0.0:1775768259.249413:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768259.249416:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:0.0:1775768259.249418:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:0.0:1775768259.249420:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:0.0:1775768259.249423:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768259.249425:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768259.249426:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768259.249427:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768259.249429:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:0.0:1775768259.249432:0:13471:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768259.249434:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768259.249436:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768259.249438:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768259.249440:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768259.249442:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775768259.249443:0:13471:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768259.249450:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fe300. 00000020:00000040:0.0:1775768259.249452:0:13471:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:0.0:1775768259.249455:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768259.249456:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768259.249458:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768259.249460:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768259.249463:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768259.249469:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220028000. 00000100:00000001:0.0:1775768259.249474:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768259.249480:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fe300 x1862022294293760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768259.249487:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768259.249490:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768259.249493:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008a1e3639] to pc [ptlrpcd_rcv+-1] req@ffff9942099fe300 x1862022294293760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768259.249527:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768259.249529:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768259.249530:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768259.249534:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fe300 x1862022294293760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768259.249539:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768259.249540:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768259.249542:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768259.249543:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768259.249545:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768259.249547:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294293760:192.168.202.156@tcp:250:kworker.0 00000100:00000001:1.0:1775768259.249550:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768259.249553:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768259.249555:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768259.249558:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768259.249562:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543fc00. 02000000:00000001:1.0:1775768259.249564:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768259.249568:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768259.249573:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c738. 00000400:00000010:1.0:1775768259.249578:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf65d8. 00000100:00000200:1.0:1775768259.249584:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294293760, portal 25 00000100:00000001:1.0:1775768259.249587:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768259.249589:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641205504 : -112966068346112 : ffff9942099fe300) 00000100:00000040:1.0:1775768259.249594:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fe300 x1862022294293760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768284 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768259.249601:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768259.249604:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768259.249606:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000100:00000200:1.0:1775768259.249608:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294293760, offset 0 00000400:00000200:1.0:1775768259.249612:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768259.249618:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768259.249622:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768259.249625:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768259.249628:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768259.249630:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768259.249632:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768259.249635:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225468:225468:256:4294967295] 192.168.202.156@tcp LPNI seq info [225464:225464:8:4294967295] 00000400:00000200:1.0:1775768259.249638:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768259.249644:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768259.249648:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768259.249651:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541300. 00000800:00000200:1.0:1775768259.249656:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768259.249658:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775768259.249662:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768259.249664:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000100:1.0:1775768259.249667:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768259.249668:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541300. 00000400:00000200:1.0:1775768259.249673:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768259.249676:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768259.249679:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000400:00000010:1.0:1775768259.249681:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000100:00000001:1.0:1775768259.249684:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768259.249686:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fe300 x1862022294293760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768284 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768259.249709:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768259.249712:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768259.249716:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775768259.249717:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768259.249719:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768259.249721:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768259.249724:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5511 00000400:00000200:0.0:1775768259.249728:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768259.249730:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768259.249732:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768259.249735:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768259.249737:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768259.249739:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768259.249741:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768259.249743:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768259.249745:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:0.0:1775768259.249746:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768259.249748:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:0.0:1775768259.249750:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:0.0:1775768259.249752:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:0.0:1775768259.249755:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768259.249757:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768259.249758:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768259.249760:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768259.249762:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00080000:0.0:1775768259.249764:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768259.249765:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768259.249767:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768259.249771:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768259.249772:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768259.249774:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775768259.249776:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775768259.249778:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768259.249780:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768259.249782:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fd880. 00000020:00000040:0.0:1775768259.249783:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:0.0:1775768259.249785:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768259.249786:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768259.249787:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768259.249788:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768259.249790:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768259.249793:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002b800. 00000100:00000001:0.0:1775768259.249796:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768259.249799:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fd880 x1862022294293888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768259.249803:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768259.249804:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768259.249806:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007f8a5f3b] to pc [ptlrpcd_rcv+-1] req@ffff9942099fd880 x1862022294293888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768259.249830:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768259.249832:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fe300 x1862022294293760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768284 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768259.249837:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768259.249838:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768259.249843:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768259.249846:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768259.249848:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4300185405 00000100:00000010:1.0:1775768259.249851:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cbaba0. 00000100:00000001:1.0:1775768259.249856:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775768259.249859:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4300185405 00000100:00000010:1.0:1775768259.249862:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba6c0. 00000100:00000001:1.0:1775768259.249863:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775768259.249865:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768259.249867:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768259.249869:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768259.249871:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768259/real 1775768259] req@ffff9942099fe300 x1862022294293760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768259.249877:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768259.249878:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768259.249879:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c738. 00000400:00000200:1.0:1775768259.249881:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf65d8 00000400:00000010:1.0:1775768259.249882:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf65d8. 00000100:00000001:1.0:1775768259.249884:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768259.249886:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fe300 x1862022294293760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768259.249891:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fe300 x1862022294293760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768259.249895:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768259.249896:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768259.249898:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768259.249899:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768259.249901:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fe300 x1862022294293760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768259.249907:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768259.249911:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fe300 x1862022294293760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768284 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768259.249917:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768259.249920:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768259.249922:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768259.249923:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768259.249925:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768259.249927:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768259.249929:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768259.249933:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768259.249935:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768259.249939:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fe300 x1862022294293760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768284 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768259.249947:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294293760:192.168.202.156@tcp:250:kworker.0 00000100:00000001:1.0:1775768259.249952:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768259.249953:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768259.249956:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fe300 x1862022294293760/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768284 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768259.249960:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768259.249962:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768259.249964:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543fc00. 02000000:00000001:1.0:1775768259.249966:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768259.249967:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768259.249968:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:1.0:1775768259.249970:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768259.249971:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028000. 02000000:00000001:1.0:1775768259.249974:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768259.249976:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768259.249977:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fe300. 00000100:00000001:1.0:1775768259.249979:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768259.249980:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768259.249981:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768259.249986:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768259.249987:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768259.249988:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768259.249989:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768259.249990:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768259.249992:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fd880 x1862022294293888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768259.249996:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768259.249997:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768259.249998:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768259.249999:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768259.250001:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768259.250002:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294293888:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768259.250004:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768259.250005:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768259.250006:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768259.250007:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768259.250009:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543fc00. 02000000:00000001:1.0:1775768259.250010:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768259.250012:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768259.250014:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c738. 00000400:00000010:1.0:1775768259.250016:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf65d8. 00000100:00000200:1.0:1775768259.250018:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294293888, portal 10 00000100:00000001:1.0:1775768259.250019:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768259.250020:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641202816 : -112966068348800 : ffff9942099fd880) 00000100:00000040:1.0:1775768259.250024:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fd880 x1862022294293888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768284 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768259.250028:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768259.250029:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768259.250030:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000100:00000200:1.0:1775768259.250032:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294293888, offset 0 00000400:00000200:1.0:1775768259.250034:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768259.250037:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768259.250039:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768259.250040:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768259.250042:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768259.250044:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768259.250045:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768259.250047:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225469:225469:256:4294967295] 192.168.202.156@tcp LPNI seq info [225465:225465:8:4294967295] 00000400:00000200:1.0:1775768259.250049:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768259.250053:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768259.250056:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768259.250058:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541300. 00000800:00000200:1.0:1775768259.250061:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768259.250062:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775768259.250065:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768259.250067:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000100:1.0:1775768259.250070:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768259.250072:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541300. 00000400:00000200:1.0:1775768259.250074:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768259.250077:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768259.250080:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000400:00000010:1.0:1775768259.250082:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000100:00000001:1.0:1775768259.250084:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768259.250087:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fd880 x1862022294293888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768284 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768259.250094:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768259.250096:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fd880 x1862022294293888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768284 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768259.250108:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768259.250110:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768259.250112:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768259.250116:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:1.0:1775768259.250119:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768259.250121:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300184082 00000100:00000001:1.0:1775768259.250122:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300185405 00000100:00000001:1.0:1775768259.250124:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300184082 00000100:00000001:1.0:1775768259.250125:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300185405 00000100:00000001:1.0:1775768259.250126:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768259.250128:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768259.250129:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768259.250131:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768259/real 1775768259] req@ffff9942099fd880 x1862022294293888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768259.250135:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768259.250136:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768259.250137:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c738. 00000400:00000200:1.0:1775768259.250139:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf65d8 00000400:00000010:1.0:1775768259.250140:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf65d8. 00000100:00000001:1.0:1775768259.250141:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768259.250143:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fd880 x1862022294293888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768259.250147:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fd880 x1862022294293888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768259.250151:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768259.250152:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768259.250153:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768259.250153:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768259.250176:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fd880 x1862022294293888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768259.250182:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768259.250186:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fd880 x1862022294293888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768284 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768259.250193:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768259.250195:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768259.250197:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768259.250198:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768259.250200:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768259.250202:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768259.250203:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768259.250206:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768259.250207:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768259.250209:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fd880 x1862022294293888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768284 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768259.250214:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294293888:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768259.250216:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768259.250217:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768259.250219:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fd880 x1862022294293888/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768284 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768259.250222:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768259.250224:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768259.250225:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543fc00. 02000000:00000001:1.0:1775768259.250226:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768259.250227:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768259.250228:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:1.0:1775768259.250229:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768259.250230:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002b800. 02000000:00000001:1.0:1775768259.250232:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768259.250233:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768259.250234:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fd880. 00000100:00000001:1.0:1775768259.250235:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768259.250236:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768259.250237:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768259.250239:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768259.250240:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768259.250241:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768259.250242:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768259.250282:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768259.250284:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768259.250286:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775768259.250287:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768259.250288:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768259.250290:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768259.250291:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5511 00000400:00000200:0.0:1775768259.250294:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768259.250296:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768259.250298:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768259.250299:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768259.250301:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768259.250302:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768259.250303:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768259.250304:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768259.250305:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:0.0:1775768259.250306:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768259.250307:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:0.0:1775768259.250308:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:0.0:1775768259.250310:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:0.0:1775768259.250312:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768259.250313:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768259.250314:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768259.250315:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768259.250316:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:0.0:1775768259.250318:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768259.250319:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768259.250320:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768259.250322:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768259.250323:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768259.250324:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775768259.250325:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775768259.250327:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768259.250328:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768259.250330:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ff480. 00000020:00000040:0.0:1775768259.250331:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:0.0:1775768259.250332:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768259.250333:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768259.250334:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768259.250335:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768259.250336:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768259.250339:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220028400. 00000100:00000001:0.0:1775768259.250341:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768259.250344:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ff480 x1862022294294016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768259.250348:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768259.250350:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768259.250352:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e73dcd12] to pc [ptlrpcd_rcv+-1] req@ffff9942099ff480 x1862022294294016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768259.250357:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768259.250359:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768259.250361:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775768259.250362:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768259.250362:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768259.250364:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768259.250365:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5511 00000400:00000200:0.0:1775768259.250367:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768259.250368:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768259.250369:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768259.250370:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768259.250371:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768259.250372:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768259.250373:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768259.250374:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768259.250375:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:0.0:1775768259.250376:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768259.250377:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:0.0:1775768259.250379:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:0.0:1775768259.250380:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:0.0:1775768259.250382:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768259.250384:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768259.250395:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768259.250397:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768259.250399:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:0.0:1775768259.250402:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768259.250404:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768259.250405:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768259.250408:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768259.250409:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768259.250411:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775768259.250413:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:0.0:1775768259.250416:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768259.250417:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768259.250420:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fd180. 00000020:00000040:0.0:1775768259.250423:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:0.0:1775768259.250425:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768259.250426:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768259.250428:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768259.250429:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768259.250431:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768259.250434:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002a800. 00000100:00000001:0.0:1775768259.250437:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768259.250441:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fd180 x1862022294294144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768259.250446:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768259.250447:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768259.250449:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004ec16d17] to pc [ptlrpcd_rcv+-1] req@ffff9942099fd180 x1862022294294144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768259.250454:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768259.250455:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768259.250457:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775768259.250458:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768259.250459:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768259.250461:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768259.250462:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5511 00000400:00000200:0.0:1775768259.250465:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768259.250466:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768259.250467:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768259.250469:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768259.250470:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768259.250471:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768259.250473:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768259.250474:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768259.250475:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:0.0:1775768259.250476:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768259.250477:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:0.0:1775768259.250478:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:0.0:1775768259.250479:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:0.0:1775768259.250481:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768259.250482:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768259.250482:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768259.250483:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768259.250485:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:0.0:1775768259.250486:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768259.250487:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768259.250488:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768259.250489:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768259.250490:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768259.250491:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775768259.250493:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:0.0:1775768259.250494:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768259.250495:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768259.250496:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fed80. 00000020:00000040:0.0:1775768259.250497:0:13471:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:0.0:1775768259.250498:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768259.250499:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768259.250500:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768259.250501:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768259.250501:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768259.250503:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220029400. 00000100:00000001:0.0:1775768259.250505:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768259.250507:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fed80 x1862022294294272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768259.250515:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768259.250516:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768259.250517:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000c09b4d8] to pc [ptlrpcd_rcv+-1] req@ffff9942099fed80 x1862022294294272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768259.250521:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768259.250524:0:13471:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5521) 00000100:00000001:1.0:1775768259.250574:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768259.250576:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768259.250578:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768259.250582:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ff480 x1862022294294016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768259.250589:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768259.250590:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768259.250593:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768259.250594:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768259.250596:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768259.250599:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294294016:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768259.250602:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768259.250603:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768259.250605:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768259.250606:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768259.250608:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543fc00. 02000000:00000001:1.0:1775768259.250609:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768259.250611:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768259.250613:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c738. 00000400:00000010:1.0:1775768259.250615:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf65d8. 00000100:00000200:1.0:1775768259.250617:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294294016, portal 10 00000100:00000001:1.0:1775768259.250618:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768259.250619:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641209984 : -112966068341632 : ffff9942099ff480) 00000100:00000040:1.0:1775768259.250622:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ff480 x1862022294294016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768284 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768259.250630:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768259.250631:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768259.250632:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000100:00000200:1.0:1775768259.250633:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294294016, offset 0 00000400:00000200:1.0:1775768259.250636:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768259.250638:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768259.250640:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768259.250642:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768259.250643:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768259.250645:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768259.250646:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768259.250648:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225470:225470:256:4294967295] 192.168.202.156@tcp LPNI seq info [225466:225466:8:4294967295] 00000400:00000200:1.0:1775768259.250650:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768259.250654:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768259.250656:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768259.250658:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541300. 00000800:00000200:1.0:1775768259.250660:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768259.250662:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775768259.250665:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768259.250666:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000100:1.0:1775768259.250669:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768259.250671:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541300. 00000400:00000200:1.0:1775768259.250673:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768259.250675:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768259.250676:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000400:00000010:1.0:1775768259.250677:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000100:00000001:1.0:1775768259.250679:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768259.250681:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ff480 x1862022294294016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768284 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768259.250685:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768259.250687:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ff480 x1862022294294016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768284 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768259.250694:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768259.250695:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768259.250697:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768259.250699:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:1.0:1775768259.250700:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768259.250702:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300184075 00000100:00000001:1.0:1775768259.250703:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300185405 00000100:00000001:1.0:1775768259.250705:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300184075 00000100:00000001:1.0:1775768259.250706:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300185405 00000100:00000001:1.0:1775768259.250707:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768259.250708:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768259.250709:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768259.250712:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768259/real 1775768259] req@ffff9942099ff480 x1862022294294016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768259.250718:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768259.250719:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768259.250721:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c738. 00000400:00000200:1.0:1775768259.250723:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf65d8 00000400:00000010:1.0:1775768259.250724:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf65d8. 00000100:00000001:1.0:1775768259.250726:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768259.250729:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ff480 x1862022294294016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768259.250736:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ff480 x1862022294294016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768259.250742:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768259.250743:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768259.250745:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768259.250747:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768259.250750:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ff480 x1862022294294016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768259.250757:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768259.250761:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ff480 x1862022294294016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768284 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768259.250766:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768259.250768:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768259.250769:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768259.250770:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768259.250771:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768259.250773:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768259.250774:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768259.250776:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768259.250778:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768259.250780:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ff480 x1862022294294016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768284 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768259.250784:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294294016:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768259.250787:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768259.250788:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fd180 x1862022294294144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768259.250792:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768259.250793:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768259.250794:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768259.250795:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768259.250796:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768259.250798:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294294144:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768259.250800:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768259.250801:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768259.250802:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768259.250803:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768259.250805:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543f800. 02000000:00000001:1.0:1775768259.250806:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768259.250809:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768259.250810:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c738. 00000400:00000010:1.0:1775768259.250812:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf65d8. 00000100:00000200:1.0:1775768259.250814:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294294144, portal 4 00000100:00000001:1.0:1775768259.250815:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768259.250816:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641201024 : -112966068350592 : ffff9942099fd180) 00000100:00000040:1.0:1775768259.250819:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fd180 x1862022294294144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768284 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768259.250823:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768259.250824:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768259.250825:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000100:00000200:1.0:1775768259.250826:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294294144, offset 0 00000400:00000200:1.0:1775768259.250828:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768259.250831:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768259.250832:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768259.250834:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768259.250835:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768259.250837:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768259.250838:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768259.250840:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225471:225471:256:4294967295] 192.168.202.156@tcp LPNI seq info [225467:225467:8:4294967295] 00000400:00000200:1.0:1775768259.250842:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768259.250846:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768259.250849:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768259.250850:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541300. 00000800:00000200:1.0:1775768259.250853:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768259.250854:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775768259.250857:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768259.250858:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000100:1.0:1775768259.250860:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768259.250862:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541300. 00000400:00000200:1.0:1775768259.250863:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768259.250865:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768259.250867:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000400:00000010:1.0:1775768259.250868:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000100:00000001:1.0:1775768259.250869:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768259.250871:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fd180 x1862022294294144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768284 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768259.250875:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768259.250877:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fd180 x1862022294294144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768284 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768259.250881:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768259.250881:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768259.250883:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768259.250884:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:1.0:1775768259.250886:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768259.250887:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300184098 00000100:00000001:1.0:1775768259.250888:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300185405 00000100:00000001:1.0:1775768259.250889:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300184098 00000100:00000001:1.0:1775768259.250891:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300185405 00000100:00000001:1.0:1775768259.250892:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768259.250893:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768259.250894:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768259.250896:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768259/real 1775768259] req@ffff9942099fd180 x1862022294294144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768259.250901:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768259.250902:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768259.250904:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c738. 00000400:00000200:1.0:1775768259.250906:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf65d8 00000400:00000010:1.0:1775768259.250907:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf65d8. 00000100:00000001:1.0:1775768259.250909:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768259.250912:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fd180 x1862022294294144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768259.250919:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fd180 x1862022294294144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768259.250925:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768259.250926:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768259.250928:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768259.250929:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768259.250933:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fd180 x1862022294294144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768259.250939:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768259.250943:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fd180 x1862022294294144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768284 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768259.250949:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768259.250951:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768259.250952:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768259.250953:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768259.250954:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768259.250957:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768259.250958:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768259.250960:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768259.250962:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768259.250964:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fd180 x1862022294294144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768284 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768259.250968:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294294144:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768259.250970:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768259.250972:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fed80 x1862022294294272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768259.250976:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768259.250977:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768259.250978:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768259.250979:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768259.250980:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768259.250982:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294294272:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768259.250983:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768259.250984:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768259.250985:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768259.250986:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768259.250988:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543d800. 02000000:00000001:1.0:1775768259.250989:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768259.250991:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768259.250992:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c738. 00000400:00000010:1.0:1775768259.250994:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf65d8. 00000100:00000200:1.0:1775768259.250996:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294294272, portal 4 00000100:00000001:1.0:1775768259.250997:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768259.250998:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641208192 : -112966068343424 : ffff9942099fed80) 00000100:00000040:1.0:1775768259.251001:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fed80 x1862022294294272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768284 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768259.251005:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768259.251006:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768259.251007:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000100:00000200:1.0:1775768259.251008:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294294272, offset 0 00000400:00000200:1.0:1775768259.251010:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768259.251013:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768259.251014:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768259.251016:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768259.251018:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768259.251019:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768259.251020:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768259.251022:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225472:225472:256:4294967295] 192.168.202.156@tcp LPNI seq info [225468:225468:8:4294967295] 00000400:00000200:1.0:1775768259.251025:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768259.251028:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768259.251031:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768259.251033:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541300. 00000800:00000200:1.0:1775768259.251035:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768259.251036:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775768259.251039:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768259.251040:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 38 secs later) 00000800:00000100:1.0:1775768259.251043:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768259.251044:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541300. 00000400:00000200:1.0:1775768259.251046:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768259.251047:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768259.251049:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000400:00000010:1.0:1775768259.251050:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000100:00000001:1.0:1775768259.251051:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768259.251053:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fed80 x1862022294294272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768284 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768259.251057:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768259.251059:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fed80 x1862022294294272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768284 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768259.251062:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768259.251063:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768259.251064:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768259.251066:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:1.0:1775768259.251068:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768259.251069:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300184101 00000100:00000001:1.0:1775768259.251071:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300185405 00000100:00000001:1.0:1775768259.251073:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300184101 00000100:00000001:1.0:1775768259.251075:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300185405 00000100:00000001:1.0:1775768259.251077:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768259.251079:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768259.251081:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768259.251083:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768259/real 1775768259] req@ffff9942099fed80 x1862022294294272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768259.251091:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768259.251092:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768259.251094:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c738. 00000400:00000200:1.0:1775768259.251096:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf65d8 00000400:00000010:1.0:1775768259.251097:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf65d8. 00000100:00000001:1.0:1775768259.251099:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768259.251102:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fed80 x1862022294294272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768259.251110:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fed80 x1862022294294272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768259.251116:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768259.251117:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768259.251119:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768259.251120:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768259.251123:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fed80 x1862022294294272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768259.251127:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768259.251129:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fed80 x1862022294294272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768284 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768259.251132:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768259.251134:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768259.251135:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768259.251136:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768259.251137:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768259.251139:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768259.251140:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768259.251142:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768259.251143:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768259.251145:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fed80 x1862022294294272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768284 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768259.251150:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294294272:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768259.251151:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768259.251152:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768259.251168:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ff480 x1862022294294016/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768284 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768259.251172:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768259.251173:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768259.251174:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543fc00. 02000000:00000001:1.0:1775768259.251175:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768259.251176:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768259.251177:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:1.0:1775768259.251178:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768259.251179:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028400. 02000000:00000001:1.0:1775768259.251181:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768259.251182:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768259.251183:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ff480. 00000100:00000001:1.0:1775768259.251184:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768259.251185:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768259.251186:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768259.251188:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fd180 x1862022294294144/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768284 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768259.251192:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768259.251192:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768259.251193:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543f800. 02000000:00000001:1.0:1775768259.251194:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768259.251195:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768259.251195:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:1.0:1775768259.251196:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768259.251197:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002a800. 02000000:00000001:1.0:1775768259.251199:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768259.251199:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768259.251200:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fd180. 00000100:00000001:1.0:1775768259.251201:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768259.251202:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768259.251203:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768259.251204:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fed80 x1862022294294272/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768284 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768259.251208:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768259.251208:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768259.251209:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543d800. 02000000:00000001:1.0:1775768259.251210:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768259.251210:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768259.251211:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:1.0:1775768259.251212:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768259.251213:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029400. 02000000:00000001:1.0:1775768259.251214:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768259.251214:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768259.251215:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fed80. 00000100:00000001:1.0:1775768259.251216:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768259.251216:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768259.251217:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768259.251220:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768259.251220:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768259.251221:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768259.251222:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768259.313216:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768259.313220:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768259.313223:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768259.313224:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768259.313226:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768259.313227:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768259.313229:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768259.313230:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768259.313231:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768259.313232:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768259.313281:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768259.313290:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768259.313294:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768259.377261:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768259.377267:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768259.377268:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768259.377274:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768259.377278:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768259.377281:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768259.377284:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768259.377286:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768259.377289:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768260.273274:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768260.273274:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768260.273274:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768260.273281:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768260.273282:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768260.273282:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768260.273284:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768260.273286:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768260.273286:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768260.273292:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:2.0:1775768260.273292:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775768260.273292:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768260.273299:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768260.273299:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768260.273304:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768260.273305:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768260.273309:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768260.273310:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768260.273314:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768260.273314:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768260.273320:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768260.273320:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768260.273324:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768260.273327:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768260.273328:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768260.273331:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768260.273334:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768260.337255:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768260.337261:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768260.337266:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768260.337268:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768260.337271:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768260.337273:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768260.337277:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768260.337278:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768260.337281:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768260.337282:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768260.337327:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768260.337330:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768260.337332:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768260.401221:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768260.401227:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768260.401229:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768260.401233:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768260.401235:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768260.401238:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768260.401240:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768260.401242:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768260.401244:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768260.593273:0:15036:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4300185405, at 4300185408 00000100:00000040:1.0:1775768260.593273:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4300185405, at 4300185408 00000100:00000010:0.0:1775768260.593286:0:15036:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cbaba0. 00000100:00000010:1.0:1775768260.593288:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cba6c0. 00000100:00000001:1.0:1775768261.297281:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768261.297281:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768261.297288:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768261.297289:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768261.297295:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768261.297295:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768261.297296:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768261.297299:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768261.297300:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768261.297308:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:1.0:1775768261.297310:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:2.0:1775768261.297313:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768261.297314:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768261.297317:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768261.297319:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768261.297319:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768261.297322:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768261.297325:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768261.297326:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768261.297333:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768261.297336:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768261.297339:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768261.297343:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768261.297346:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768261.297347:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768261.297350:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768261.297353:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768261.361295:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768261.361301:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768261.361306:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768261.361307:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768261.361310:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768261.361311:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768261.361315:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768261.361316:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768261.361319:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768261.361319:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768261.361375:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775768261.361378:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775768261.361382:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768261.425310:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768261.425319:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768261.425322:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768261.425329:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768261.425334:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768261.425339:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768261.425343:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768261.425346:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768261.425350:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768262.321276:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768262.321276:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768262.321283:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768262.321283:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768262.321284:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768262.321287:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768262.321288:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768262.321296:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775768262.321296:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768262.321298:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768262.321304:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768262.321305:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768262.321305:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768262.321312:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768262.321312:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768262.321313:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768262.321318:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768262.321320:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768262.321321:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768262.321326:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768262.321333:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768262.321333:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768262.321336:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768262.321339:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768262.321341:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768262.321345:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768262.321349:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768262.385303:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768262.385311:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768262.385314:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768262.385318:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768262.385324:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768262.385327:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768262.385329:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768262.385332:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768262.385337:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768262.385338:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768262.385343:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768262.385344:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768262.385348:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768262.385349:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768262.385353:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768262.385354:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768262.385447:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768262.385451:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768262.385455:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768262.449312:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768262.449321:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768262.449323:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768262.449330:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768262.449335:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768262.449340:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768262.449343:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768262.449346:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768262.449350:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768263.345289:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768263.345295:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768263.345295:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768263.345298:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768263.345303:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768263.345304:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768263.345305:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768263.345306:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768263.345313:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768263.345315:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775768263.345315:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768263.345322:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768263.345322:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:2.0:1775768263.345325:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768263.345328:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768263.345328:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768263.345336:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768263.345338:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768263.345340:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768263.345343:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768263.345343:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768263.345345:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768263.345367:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768263.345370:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768263.345371:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768263.345376:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768263.345380:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768263.409276:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768263.409281:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768263.409285:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768263.409286:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768263.409291:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768263.409292:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768263.409295:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768263.409296:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768263.409298:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768263.409299:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768263.409364:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768263.409367:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768263.409369:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768263.473327:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768263.473335:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768263.473338:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768263.473368:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768263.473374:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768263.473379:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768263.473382:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768263.473385:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768263.473388:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768264.369239:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768264.369239:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768264.369239:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768264.369245:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768264.369246:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768264.369246:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768264.369248:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768264.369248:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768264.369249:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768264.369255:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:2.0:1775768264.369255:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:3.0:1775768264.369255:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768264.369260:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768264.369261:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768264.369261:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768264.369265:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768264.369265:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768264.369265:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768264.369269:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768264.369272:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768264.369274:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768264.369275:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768264.369277:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768264.369279:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768264.369280:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768264.369282:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768264.369285:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768264.369298:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768264.369302:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775768264.369305:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775768264.369307:0:36:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775768264.369309:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775768264.369314:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5516 00000400:00000200:2.0:1775768264.369322:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768264.369326:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775768264.369328:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775768264.369331:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768264.369333:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768264.369335:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768264.369353:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775768264.369356:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775768264.369357:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:2.0:1775768264.369359:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775768264.369361:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:2.0:1775768264.369363:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:2.0:1775768264.369364:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:2.0:1775768264.369366:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768264.369367:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768264.369368:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768264.369369:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775768264.369371:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:2.0:1775768264.369373:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775768264.369375:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768264.369377:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775768264.369379:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768264.369381:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768264.369383:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775768264.369384:0:36:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775768264.369389:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dff100. 00000020:00000040:2.0:1775768264.369392:0:36:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:2.0:1775768264.369395:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775768264.369396:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768264.369398:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775768264.369400:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775768264.369404:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775768264.369409:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1fc00. 00000100:00000001:2.0:1775768264.369415:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768264.369423:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dff100 x1862022294294400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775768264.369430:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775768264.369434:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768264.369438:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3e64ae1] to pc [ptlrpcd_rcv+-1] req@ffff994207dff100 x1862022294294400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768264.369453:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768264.369456:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768264.369458:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775768264.369459:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775768264.369461:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775768264.369463:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775768264.369464:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5516 00000400:00000200:2.0:1775768264.369467:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768264.369468:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775768264.369470:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775768264.369472:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768264.369474:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768264.369476:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768264.369478:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775768264.369480:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775768264.369481:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:2.0:1775768264.369483:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775768264.369484:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:2.0:1775768264.369485:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:2.0:1775768264.369487:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:2.0:1775768264.369488:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768264.369490:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768264.369490:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768264.369491:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768264.369494:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:1.0:1775768264.369495:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000040:2.0:1775768264.369495:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00000001:1.0:1775768264.369498:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00080000:2.0:1775768264.369498:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775768264.369502:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768264.369503:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:1.0:1775768264.369508:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dff100 x1862022294294400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1775768264.369508:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768264.369512:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768264.369515:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775768264.369519:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1775768264.369520:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768264.369523:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1775768264.369524:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768264.369527:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768264.369529:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000010:2.0:1775768264.369530:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfdc00. 02000000:00000001:1.0:1775768264.369531:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:2.0:1775768264.369533:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:1.0:1775768264.369534:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768264.369537:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00100000:1.0:1775768264.369539:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294294400:192.168.202.156@tcp:250:kworker.0 02000000:00000001:2.0:1775768264.369539:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768264.369541:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775768264.369544:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1775768264.369545:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768264.369546:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768264.369549:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000010:2.0:1775768264.369550:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1ec00. 02000000:00000001:1.0:1775768264.369551:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768264.369554:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1775768264.369554:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775768264.369558:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543f800. 00000100:00100000:2.0:1775768264.369560:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfdc00 x1862022294294528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1775768264.369561:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768264.369569:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1775768264.369570:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768264.369572:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775768264.369575:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c738. 00000100:00000040:2.0:1775768264.369577:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003e25603c] to pc [ptlrpcd_rcv+-1] req@ffff994207dfdc00 x1862022294294528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1775768264.369582:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf65d8. 00000100:00000001:2.0:1775768264.369584:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768264.369587:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:1.0:1775768264.369588:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294294400, portal 25 00000100:00080000:2.0:1775768264.369589:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775768264.369592:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768264.369593:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775768264.369595:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611848960 : -112966097702656 : ffff994207dff100) 00000100:00080000:2.0:1775768264.369595:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775768264.369598:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775768264.369601:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5516 00000100:00000040:1.0:1775768264.369603:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dff100 x1862022294294400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768289 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1775768264.369607:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768264.369613:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:1.0:1775768264.369615:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1775768264.369616:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775768264.369620:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000100:00000040:2.0:1775768264.369620:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000010:1.0:1775768264.369623:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000100:00000001:2.0:1775768264.369624:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768264.369627:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000200:1.0:1775768264.369628:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294294400, offset 0 00000100:00000001:2.0:1775768264.369629:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775768264.369632:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775768264.369633:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:2.0:1775768264.369634:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:1.0:1775768264.369637:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000020:00000040:2.0:1775768264.369637:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:2.0:1775768264.369641:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:2.0:1775768264.369643:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:2.0:1775768264.369645:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000200:1.0:1775768264.369648:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768264.369649:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768264.369651:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768264.369653:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000200:1.0:1775768264.369655:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000020:00000040:2.0:1775768264.369656:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:2.0:1775768264.369661:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000400:00000200:1.0:1775768264.369662:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1775768264.369664:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768264.369665:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1775768264.369667:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 02000000:00000001:2.0:1775768264.369668:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768264.369670:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768264.369671:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000100:00000001:2.0:1775768264.369672:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:1.0:1775768264.369675:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000008:00100000:2.0:1775768264.369675:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775768264.369678:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768264.369681:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225473:225473:256:4294967295] 192.168.202.156@tcp LPNI seq info [225469:225469:8:4294967295] 00000100:00000001:2.0:1775768264.369681:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775768264.369685:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfc700. 00000400:00000200:1.0:1775768264.369687:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000040:2.0:1775768264.369689:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:2.0:1775768264.369691:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775768264.369693:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768264.369694:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775768264.369695:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775768264.369698:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768264.369701:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 02000000:00000010:2.0:1775768264.369703:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1f000. 00000100:00000001:2.0:1775768264.369706:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775768264.369709:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00100000:2.0:1775768264.369713:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfc700 x1862022294294656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000010:1.0:1775768264.369714:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541300. 00000100:00000001:2.0:1775768264.369720:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775768264.369722:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775768264.369725:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000040:2.0:1775768264.369729:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a176dd1] to pc [ptlrpcd_rcv+-1] req@ffff994207dfc700 x1862022294294656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1775768264.369730:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1775768264.369740:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:2.0:1775768264.369745:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775768264.369745:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000100:1.0:1775768264.369749:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000100:00080000:2.0:1775768264.369750:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000010:1.0:1775768264.369754:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541300. 00000100:00080000:2.0:1775768264.369755:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1775768264.369757:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775768264.369760:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1775768264.369761:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:2.0:1775768264.369764:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775768264.369769:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5516 00000400:00000200:1.0:1775768264.369773:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768264.369777:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768264.369781:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000400:00000200:2.0:1775768264.369782:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775768264.369787:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000400:00000010:1.0:1775768264.369788:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000100:00000040:2.0:1775768264.369791:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768264.369794:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1775768264.369794:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768264.369797:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768264.369799:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000200:1.0:1775768264.369803:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dff100 x1862022294294400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768289 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1775768264.369803:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775768264.369804:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:2.0:1775768264.369808:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775768264.369810:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:2.0:1775768264.369815:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:2.0:1775768264.369819:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:2.0:1775768264.369824:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768264.369825:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1775768264.369826:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768264.369830:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000040:1.0:1775768264.369834:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dff100 x1862022294294400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768289 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768264.369834:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775768264.369838:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:2.0:1775768264.369845:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775768264.369850:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768264.369851:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768264.369852:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1775768264.369856:0:2399:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:2.0:1775768264.369857:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768264.369861:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768264.369863:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775768264.369864:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1775768264.369865:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:2.0:1775768264.369869:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768264.369873:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768264.369874:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000010:2.0:1775768264.369878:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfe300. 00000100:00000001:1.0:1775768264.369879:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4300190525 00000020:00000040:2.0:1775768264.369884:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00000100:00000010:1.0:1775768264.369887:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba6c0. 02000000:00000001:2.0:1775768264.369888:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775768264.369890:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768264.369892:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775768264.369893:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775768264.369895:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768264.369896:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775768264.369903:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4300190525 02000000:00000010:2.0:1775768264.369906:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1f400. 00000100:00000010:1.0:1775768264.369910:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba3c0. 00000100:00000001:2.0:1775768264.369910:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768264.369916:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00100000:2.0:1775768264.369917:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfe300 x1862022294294784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775768264.369923:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775768264.369924:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768264.369926:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768264.369930:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768264.369932:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c661fef] to pc [ptlrpcd_rcv+-1] req@ffff994207dfe300 x1862022294294784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768264.369934:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768264.369944:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768264/real 1775768264] req@ffff994207dff100 x1862022294294400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768264.369944:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768264.369949:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768264.369958:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1775768264.369961:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000400:00000001:1.0:1775768264.369964:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:2.0:1775768264.369964:0:36:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000001:1.0:1775768264.369968:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1775768264.369969:0:36:0:(import.c:484:import_select_connection()) Process entered 00000400:00000010:1.0:1775768264.369973:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c738. 00000100:00080000:2.0:1775768264.369973:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5516 00000400:00000200:1.0:1775768264.369979:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf65d8 00000400:00000200:2.0:1775768264.369981:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768264.369982:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf65d8. 00000400:00000200:2.0:1775768264.369986:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:1.0:1775768264.369989:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00080000:2.0:1775768264.369989:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775768264.369994:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000200:1.0:1775768264.369997:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dff100 x1862022294294400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768264.369998:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768264.370003:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768264.370006:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775768264.370008:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775768264.370009:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:2.0:1775768264.370011:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775768264.370012:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:2.0:1775768264.370017:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:2.0:1775768264.370022:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000200:1.0:1775768264.370027:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dff100 x1862022294294400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775768264.370029:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768264.370033:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768264.370036:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768264.370040:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775768264.370043:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:2.0:1775768264.370047:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775768264.370048:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768264.370049:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768264.370051:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1775768264.370052:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1775768264.370054:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768264.370056:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:2.0:1775768264.370057:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768264.370060:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775768264.370061:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:2.0:1775768264.370061:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:2.0:1775768264.370065:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768264.370067:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768264.370070:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dff100 x1862022294294400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1775768264.370071:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfed80. 00000020:00000040:2.0:1775768264.370074:0:36:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:2.0:1775768264.370078:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775768264.370079:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768264.370083:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775768264.370085:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775768264.370087:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775768264.370090:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1d000. 00000100:00000001:1.0:1775768264.370091:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768264.370094:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768264.370100:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfed80 x1862022294294912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:1.0:1775768264.370102:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dff100 x1862022294294400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768289 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768264.370112:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775768264.370115:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768264.370120:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb9b9eae] to pc [ptlrpcd_rcv+-1] req@ffff994207dfed80 x1862022294294912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768264.370123:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768264.370131:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768264.370131:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768264.370136:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768264.370138:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768264.370138:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5526) 00000100:00000001:1.0:1775768264.370143:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768264.370145:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768264.370148:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768264.370154:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768264.370178:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768264.370184:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dff100 x1862022294294400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768289 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768264.370194:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294294400:192.168.202.156@tcp:250:kworker.0 00000100:00000001:1.0:1775768264.370199:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768264.370201:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768264.370203:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dff100 x1862022294294400/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768289 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768264.370210:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768264.370212:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768264.370215:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543f800. 02000000:00000001:1.0:1775768264.370217:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768264.370219:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768264.370220:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:1.0:1775768264.370222:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768264.370225:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1fc00. 02000000:00000001:1.0:1775768264.370228:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768264.370231:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768264.370232:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dff100. 00000100:00000001:1.0:1775768264.370235:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768264.370236:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768264.370238:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768264.370244:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768264.370246:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768264.370248:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768264.370249:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768264.370250:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768264.370254:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfdc00 x1862022294294528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768264.370260:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768264.370261:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768264.370264:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768264.370265:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768264.370267:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768264.370270:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294294528:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768264.370272:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768264.370274:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768264.370275:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768264.370277:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768264.370281:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543f800. 02000000:00000001:1.0:1775768264.370282:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768264.370286:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768264.370288:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c738. 00000400:00000010:1.0:1775768264.370291:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf65d8. 00000100:00000200:1.0:1775768264.370294:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294294528, portal 10 00000100:00000001:1.0:1775768264.370296:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768264.370298:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611843584 : -112966097708032 : ffff994207dfdc00) 00000100:00000040:1.0:1775768264.370303:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfdc00 x1862022294294528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768289 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768264.370309:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768264.370311:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768264.370313:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000100:00000200:1.0:1775768264.370315:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294294528, offset 0 00000400:00000200:1.0:1775768264.370328:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768264.370333:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768264.370348:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768264.370351:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768264.370354:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768264.370357:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768264.370359:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768264.370362:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225474:225474:256:4294967295] 192.168.202.156@tcp LPNI seq info [225470:225470:8:4294967295] 00000400:00000200:1.0:1775768264.370366:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768264.370372:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768264.370377:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768264.370380:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541300. 00000800:00000200:1.0:1775768264.370384:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768264.370387:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1775768264.370391:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768264.370393:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000100:1.0:1775768264.370396:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768264.370398:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541300. 00000400:00000200:1.0:1775768264.370400:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768264.370404:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768264.370407:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000400:00000010:1.0:1775768264.370408:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000100:00000001:1.0:1775768264.370411:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768264.370414:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfdc00 x1862022294294528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768289 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768264.370421:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768264.370424:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfdc00 x1862022294294528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768289 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768264.370429:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768264.370431:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768264.370434:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768264.370438:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:1.0:1775768264.370441:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768264.370443:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300189203 00000100:00000001:1.0:1775768264.370446:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300190525 00000100:00000001:1.0:1775768264.370448:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300189203 00000100:00000001:1.0:1775768264.370450:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300190525 00000100:00000001:1.0:1775768264.370451:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768264.370454:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768264.370455:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768264.370458:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768264/real 1775768264] req@ffff994207dfdc00 x1862022294294528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768264.370466:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768264.370467:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768264.370469:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c738. 00000400:00000200:1.0:1775768264.370471:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf65d8 00000400:00000010:1.0:1775768264.370472:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf65d8. 00000100:00000001:1.0:1775768264.370474:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768264.370477:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfdc00 x1862022294294528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768264.370485:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfdc00 x1862022294294528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768264.370490:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768264.370491:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768264.370493:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768264.370494:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768264.370497:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfdc00 x1862022294294528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768264.370503:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768264.370506:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfdc00 x1862022294294528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768289 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768264.370513:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768264.370515:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768264.370517:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768264.370518:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768264.370520:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768264.370522:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768264.370524:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768264.370527:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768264.370529:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768264.370534:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfdc00 x1862022294294528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768289 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768264.370541:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294294528:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768264.370545:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768264.370548:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfc700 x1862022294294656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768264.370553:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768264.370555:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768264.370556:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768264.370558:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768264.370560:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768264.370562:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294294656:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768264.370566:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768264.370567:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768264.370569:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768264.370570:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768264.370573:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543fc00. 02000000:00000001:1.0:1775768264.370575:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768264.370578:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768264.370580:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c738. 00000400:00000010:1.0:1775768264.370583:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf65d8. 00000100:00000200:1.0:1775768264.370585:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294294656, portal 10 00000100:00000001:1.0:1775768264.370587:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768264.370588:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611838208 : -112966097713408 : ffff994207dfc700) 00000100:00000040:1.0:1775768264.370592:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfc700 x1862022294294656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768289 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768264.370598:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768264.370600:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768264.370602:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000100:00000200:1.0:1775768264.370604:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294294656, offset 0 00000400:00000200:1.0:1775768264.370607:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768264.370611:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768264.370614:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768264.370616:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768264.370618:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768264.370620:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768264.370622:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768264.370625:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225475:225475:256:4294967295] 192.168.202.156@tcp LPNI seq info [225471:225471:8:4294967295] 00000400:00000200:1.0:1775768264.370628:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768264.370634:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768264.370637:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768264.370641:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541300. 00000800:00000200:1.0:1775768264.370645:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768264.370647:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1775768264.370650:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768264.370652:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000100:1.0:1775768264.370655:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768264.370657:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541300. 00000400:00000200:1.0:1775768264.370660:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768264.370663:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768264.370665:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000400:00000010:1.0:1775768264.370667:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000100:00000001:1.0:1775768264.370669:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768264.370673:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfc700 x1862022294294656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768289 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768264.370679:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768264.370681:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfc700 x1862022294294656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768289 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768264.370687:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768264.370688:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768264.370691:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768264.370693:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:1.0:1775768264.370696:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768264.370698:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300189195 00000100:00000001:1.0:1775768264.370700:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300190525 00000100:00000001:1.0:1775768264.370702:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300189195 00000100:00000001:1.0:1775768264.370704:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300190525 00000100:00000001:1.0:1775768264.370706:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768264.370708:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768264.370709:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768264.370713:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768264/real 1775768264] req@ffff994207dfc700 x1862022294294656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768264.370719:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768264.370720:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768264.370722:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c738. 00000400:00000200:1.0:1775768264.370724:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf65d8 00000400:00000010:1.0:1775768264.370726:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf65d8. 00000100:00000001:1.0:1775768264.370728:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768264.370731:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfc700 x1862022294294656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768264.370738:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfc700 x1862022294294656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768264.370744:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768264.370745:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768264.370747:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768264.370748:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768264.370751:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfc700 x1862022294294656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768264.370757:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768264.370760:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfc700 x1862022294294656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768289 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768264.370766:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768264.370768:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768264.370769:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768264.370770:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768264.370772:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768264.370774:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768264.370777:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768264.370780:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768264.370782:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768264.370785:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfc700 x1862022294294656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768289 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768264.370792:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294294656:192.168.202.156@tcp:38:kworker.0 00000100:00000001:1.0:1775768264.370796:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768264.370798:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfe300 x1862022294294784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768264.370803:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768264.370805:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768264.370807:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768264.370808:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768264.370810:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768264.370813:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294294784:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768264.370816:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768264.370818:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768264.370819:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768264.370820:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768264.370823:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543d000. 02000000:00000001:1.0:1775768264.370825:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768264.370829:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768264.370831:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c738. 00000400:00000010:1.0:1775768264.370834:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf65d8. 00000100:00000200:1.0:1775768264.370836:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294294784, portal 4 00000100:00000001:1.0:1775768264.370838:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768264.370839:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611845376 : -112966097706240 : ffff994207dfe300) 00000100:00000040:1.0:1775768264.370844:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfe300 x1862022294294784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768289 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768264.370850:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768264.370852:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768264.370854:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000100:00000200:1.0:1775768264.370856:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294294784, offset 0 00000400:00000200:1.0:1775768264.370860:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768264.370863:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768264.370866:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768264.370868:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768264.370871:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768264.370873:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768264.370876:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768264.370878:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225476:225476:256:4294967295] 192.168.202.156@tcp LPNI seq info [225472:225472:8:4294967295] 00000400:00000200:1.0:1775768264.370882:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768264.370888:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768264.370892:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768264.370895:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541300. 00000800:00000200:1.0:1775768264.370899:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768264.370901:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1775768264.370905:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768264.370908:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000100:1.0:1775768264.370911:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768264.370912:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541300. 00000400:00000200:1.0:1775768264.370914:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768264.370918:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768264.370920:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000400:00000010:1.0:1775768264.370922:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000100:00000001:1.0:1775768264.370924:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768264.370927:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfe300 x1862022294294784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768289 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768264.370933:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768264.370936:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfe300 x1862022294294784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768289 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768264.370942:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768264.370943:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768264.370946:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768264.370949:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:1.0:1775768264.370952:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768264.370954:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300189218 00000100:00000001:1.0:1775768264.370956:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300190525 00000100:00000001:1.0:1775768264.370958:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300189218 00000100:00000001:1.0:1775768264.370959:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300190525 00000100:00000001:1.0:1775768264.370961:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768264.370963:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768264.370964:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768264.370968:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768264/real 1775768264] req@ffff994207dfe300 x1862022294294784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768264.370975:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768264.370976:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768264.370978:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c738. 00000400:00000200:1.0:1775768264.370980:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf65d8 00000400:00000010:1.0:1775768264.370982:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf65d8. 00000100:00000001:1.0:1775768264.370984:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768264.370987:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfe300 x1862022294294784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768264.370994:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfe300 x1862022294294784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768264.371001:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768264.371002:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768264.371004:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768264.371005:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768264.371008:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfe300 x1862022294294784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768264.371015:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768264.371018:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfe300 x1862022294294784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768289 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768264.371025:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768264.371027:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768264.371028:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768264.371029:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768264.371031:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768264.371034:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768264.371036:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768264.371039:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768264.371041:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768264.371045:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfe300 x1862022294294784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768289 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768264.371052:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294294784:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768264.371055:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768264.371057:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfed80 x1862022294294912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768264.371063:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768264.371064:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768264.371066:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768264.371067:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768264.371069:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768264.371072:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294294912:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768264.371075:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768264.371077:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768264.371078:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768264.371079:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768264.371082:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543dc00. 02000000:00000001:1.0:1775768264.371084:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768264.371087:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768264.371089:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c738. 00000400:00000010:1.0:1775768264.371092:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf65d8. 00000100:00000200:1.0:1775768264.371095:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294294912, portal 4 00000100:00000001:1.0:1775768264.371096:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768264.371097:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611848064 : -112966097703552 : ffff994207dfed80) 00000100:00000040:1.0:1775768264.371102:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfed80 x1862022294294912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768289 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768264.371107:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768264.371109:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768264.371111:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00000100:00000200:1.0:1775768264.371113:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294294912, offset 0 00000400:00000200:1.0:1775768264.371117:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768264.371120:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768264.371122:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768264.371127:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768264.371129:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768264.371130:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768264.371132:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768264.371135:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225477:225477:256:4294967295] 192.168.202.156@tcp LPNI seq info [225473:225473:8:4294967295] 00000400:00000200:1.0:1775768264.371138:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768264.371143:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768264.371147:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768264.371150:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541300. 00000800:00000200:1.0:1775768264.371153:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.0:1775768264.371178:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1775768264.371183:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768264.371193:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 33 secs later) 00000800:00000100:1.0:1775768264.371197:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768264.371199:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541300. 00000400:00000200:1.0:1775768264.371202:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775768264.371205:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775768264.371208:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000400:00000010:1.0:1775768264.371209:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000100:00000001:1.0:1775768264.371211:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768264.371214:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfed80 x1862022294294912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768289 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768264.371221:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768264.371223:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfed80 x1862022294294912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768289 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768264.371228:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768264.371229:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775768264.371231:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768264.371234:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:1.0:1775768264.371237:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768264.371239:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300189222 00000100:00000001:1.0:1775768264.371241:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300190525 00000100:00000001:1.0:1775768264.371243:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300189222 00000100:00000001:1.0:1775768264.371244:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300190525 00000100:00000001:1.0:1775768264.371246:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768264.371248:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768264.371249:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775768264.371253:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768264/real 1775768264] req@ffff994207dfed80 x1862022294294912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775768264.371259:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775768264.371260:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775768264.371262:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c738. 00000400:00000200:1.0:1775768264.371264:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf65d8 00000400:00000010:1.0:1775768264.371265:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf65d8. 00000100:00000001:1.0:1775768264.371267:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775768264.371270:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfed80 x1862022294294912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775768264.371276:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfed80 x1862022294294912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768264.371281:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775768264.371283:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768264.371284:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768264.371285:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768264.371288:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfed80 x1862022294294912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768264.371293:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768264.371296:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfed80 x1862022294294912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768289 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768264.371301:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768264.371303:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768264.371305:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775768264.371306:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768264.371307:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775768264.371310:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768264.371312:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768264.371315:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768264.371317:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775768264.371319:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfed80 x1862022294294912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768289 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768264.371326:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294294912:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768264.371329:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768264.371331:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768264.371333:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfdc00 x1862022294294528/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768289 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768264.371350:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768264.371351:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768264.371353:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543f800. 02000000:00000001:1.0:1775768264.371355:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768264.371356:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768264.371358:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:1.0:1775768264.371359:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768264.371361:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1ec00. 02000000:00000001:1.0:1775768264.371364:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768264.371365:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768264.371367:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfdc00. 00000100:00000001:1.0:1775768264.371369:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768264.371370:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768264.371372:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768264.371375:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfc700 x1862022294294656/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768289 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768264.371386:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768264.371387:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768264.371389:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543fc00. 02000000:00000001:1.0:1775768264.371391:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768264.371392:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768264.371393:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:1.0:1775768264.371395:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768264.371396:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1f000. 02000000:00000001:1.0:1775768264.371397:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768264.371398:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768264.371399:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfc700. 00000100:00000001:1.0:1775768264.371401:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768264.371402:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768264.371403:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768264.371406:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfe300 x1862022294294784/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768289 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768264.371411:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768264.371412:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768264.371414:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543d000. 02000000:00000001:1.0:1775768264.371416:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768264.371416:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768264.371418:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:1.0:1775768264.371419:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768264.371420:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1f400. 02000000:00000001:1.0:1775768264.371422:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768264.371423:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768264.371424:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfe300. 00000100:00000001:1.0:1775768264.371426:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768264.371427:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768264.371428:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768264.371430:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfed80 x1862022294294912/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768289 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768264.371436:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775768264.371436:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775768264.371438:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543dc00. 02000000:00000001:1.0:1775768264.371439:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768264.371440:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775768264.371441:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:1.0:1775768264.371443:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775768264.371444:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1d000. 02000000:00000001:1.0:1775768264.371446:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775768264.371447:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775768264.371449:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfed80. 00000100:00000001:1.0:1775768264.371451:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768264.371452:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768264.371453:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768264.371457:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768264.371458:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768264.371459:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768264.371461:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768264.433255:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768264.433259:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768264.433264:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768264.433265:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768264.433267:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768264.433269:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768264.433271:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768264.433272:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768264.433274:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768264.433274:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768264.433347:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768264.433350:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768264.433353:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768264.497250:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768264.497256:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768264.497258:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768264.497263:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768264.497267:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768264.497271:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768264.497273:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768264.497275:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768264.497279:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768265.393236:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768265.393236:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768265.393244:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768265.393244:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768265.393246:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768265.393247:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768265.393254:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:2.0:1775768265.393254:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768265.393260:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768265.393260:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768265.393265:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768265.393266:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768265.393270:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768265.393273:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768265.393276:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768265.393278:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768265.393281:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768265.393283:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768265.393595:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768265.393599:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768265.393601:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768265.393605:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768265.393608:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768265.393611:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768265.393613:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768265.393615:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768265.393618:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768265.457239:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768265.457244:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768265.457247:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768265.457248:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768265.457250:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768265.457251:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768265.457253:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768265.457254:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768265.457255:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768265.457256:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768265.457291:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768265.457293:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768265.457294:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768265.521257:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768265.521263:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768265.521265:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768265.521271:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768265.521274:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768265.521277:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768265.521280:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768265.521282:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768265.521284:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768265.713244:0:15036:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4300190525, at 4300190528 00000100:00000040:1.0:1775768265.713244:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4300190525, at 4300190528 00000100:00000010:1.0:1775768265.713257:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cba3c0. 00000100:00000010:0.0:1775768265.713259:0:15036:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cba6c0. 00000100:00000001:3.0:1775768266.417216:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768266.417222:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768266.417223:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768266.417224:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768266.417225:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768266.417228:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768266.417228:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768266.417229:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768266.417233:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768266.417235:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775768266.417235:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775768266.417237:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768266.417239:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768266.417240:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768266.417243:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768266.417243:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768266.417244:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768266.417247:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768266.417247:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768266.417249:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768266.417252:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768266.417255:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768266.417257:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768266.417260:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768266.417260:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768266.417264:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768266.417267:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768266.481266:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768266.481271:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768266.481275:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768266.481276:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768266.481279:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768266.481280:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768266.481282:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768266.481283:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768266.481285:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768266.481286:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768266.481346:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768266.481349:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768266.481352:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768266.545227:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768266.545232:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768266.545234:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768266.545240:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768266.545243:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768266.545246:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768266.545247:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768266.545249:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768266.545251:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768267.441221:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768267.441228:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768267.441239:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768267.441249:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768267.441253:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768267.441257:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768267.441259:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768267.441262:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768267.441267:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768267.441277:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768267.441282:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768267.441285:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768267.441288:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768267.441290:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768267.441292:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768267.441294:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768267.441296:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768267.441297:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768267.442186:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768267.442188:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768267.442190:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768267.442191:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775768267.442193:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768267.442194:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768267.442195:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768267.442197:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768267.442198:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768267.505268:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768267.505276:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768267.505287:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768267.505291:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768267.505293:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768267.505297:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768267.505300:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768267.505304:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768267.505319:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768267.505322:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768267.505326:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768267.505328:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768267.505329:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768267.505330:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768267.505332:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768267.505333:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768267.505334:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768267.505336:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768267.505340:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768267.505342:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768267.505343:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768267.505344:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768267.505346:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768267.505347:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768267.505348:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768267.505350:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768267.505354:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768267.505355:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768267.505356:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768267.505358:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768267.505359:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768267.505360:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768267.505361:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768267.505363:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768267.505389:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768267.505401:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768267.505405:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768267.569230:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768267.569237:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768267.569239:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768267.569244:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768267.569247:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768267.569251:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768267.569253:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768267.569256:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768267.569258:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768268.465226:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768268.465231:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768268.465233:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768268.465238:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:0.0:1775768268.465241:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768268.465243:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768268.465245:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768268.465249:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768268.465249:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768268.465253:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00100000:0.0:1775768268.465255:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768268.465257:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768268.465259:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768268.465262:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768268.465263:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768268.465266:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768268.465268:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768268.465271:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768268.465286:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768268.465292:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768268.465293:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768268.465310:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768268.465315:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768268.465319:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768268.465322:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768268.465325:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768268.465328:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768268.529235:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768268.529240:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768268.529246:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768268.529248:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768268.529252:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768268.529254:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768268.529257:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768268.529258:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768268.529261:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768268.529262:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768268.529359:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768268.529366:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768268.529371:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768268.593286:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768268.593294:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768268.593309:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768268.593316:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768268.593321:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768268.593325:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768268.593328:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768268.593331:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768268.593334:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768269.489269:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768269.489275:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768269.489277:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768269.489279:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768269.489280:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768269.489281:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1775768269.489281:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768269.489295:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768269.489300:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1775768269.489303:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:0.0:1775768269.489305:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768269.489305:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768269.489308:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1775768269.489309:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1775768269.489312:0:36:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768269.489313:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768269.489316:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768269.489319:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:3.0:1775768269.489323:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768269.489327:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768269.489387:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768269.489389:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768269.489392:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768269.489400:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5521 00000400:00000200:2.0:1775768269.489411:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768269.489417:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775768269.489421:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775768269.489424:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768269.489427:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768269.489429:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768269.489432:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775768269.489436:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775768269.489437:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:2.0:1775768269.489439:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775768269.489442:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:2.0:1775768269.489444:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:2.0:1775768269.489446:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:2.0:1775768269.489449:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768269.489451:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768269.489452:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768269.489455:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775768269.489457:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:2.0:1775768269.489461:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775768269.489463:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768269.489466:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775768269.489469:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768269.489472:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768269.489475:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775768269.489477:0:36:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775768269.489486:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfce00. 00000020:00000040:2.0:1775768269.489490:0:36:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:2.0:1775768269.489494:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775768269.489495:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768269.489498:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775768269.489500:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775768269.489504:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775768269.489512:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1e400. 00000100:00000001:2.0:1775768269.489518:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768269.489528:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfce00 x1862022294295040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775768269.489536:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775768269.489540:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768269.489549:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e5a7debe] to pc [ptlrpcd_rcv+-1] req@ffff994207dfce00 x1862022294295040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768269.489570:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768269.489573:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768269.489576:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775768269.489577:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775768269.489579:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775768269.489584:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775768269.489587:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5521 00000400:00000200:2.0:1775768269.489591:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768269.489595:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775768269.489597:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775768269.489600:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768269.489602:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1775768269.489603:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1775768269.489605:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768269.489608:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775768269.489610:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775768269.489611:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:2.0:1775768269.489613:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775768269.489615:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000100:00000001:0.0:1775768269.489616:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1775768269.489617:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000001:0.0:1775768269.489619:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000001:2.0:1775768269.489621:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:0.0:1775768269.489627:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfce00 x1862022294295040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775768269.489627:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768269.489631:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768269.489636:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768269.489640:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775768269.489645:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00000001:0.0:1775768269.489647:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768269.489650:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768269.489651:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 02000000:00000001:0.0:1775768269.489656:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768269.489659:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768269.489660:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768269.489663:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768269.489664:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:0.0:1775768269.489670:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294295040:192.168.202.156@tcp:250:kworker.0 02000000:00000001:2.0:1775768269.489672:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768269.489677:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768269.489677:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768269.489681:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:0.0:1775768269.489683:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000008:00100000:2.0:1775768269.489686:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1775768269.489687:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768269.489691:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:2.0:1775768269.489692:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768269.489695:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768269.489697:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002bc00. 00000100:00000010:2.0:1775768269.489699:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dff480. 02000000:00000001:0.0:1775768269.489701:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775768269.489703:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:2.0:1775768269.489706:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000200:0.0:1775768269.489709:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:2.0:1775768269.489709:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768269.489713:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775768269.489715:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:0.0:1775768269.489716:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbc268. 02000000:00000001:2.0:1775768269.489717:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775768269.489724:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1c800. 00000400:00000010:0.0:1775768269.489725:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014ee0. 00000100:00000001:2.0:1775768269.489727:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768269.489734:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dff480 x1862022294295168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:0.0:1775768269.489740:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294295040, portal 25 00000100:00000001:2.0:1775768269.489744:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768269.489745:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768269.489748:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611840000 : -112966097711616 : ffff994207dfce00) 00000100:00000001:2.0:1775768269.489748:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768269.489755:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093a15639] to pc [ptlrpcd_rcv+-1] req@ffff994207dff480 x1862022294295168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775768269.489758:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfce00 x1862022294295040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768294 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768269.489779:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768269.489781:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1775768269.489783:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775768269.489787:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000100:00080000:2.0:1775768269.489790:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000010:0.0:1775768269.489792:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000001:2.0:1775768269.489792:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775768269.489795:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:0.0:1775768269.489797:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294295040, offset 0 00000100:00000001:2.0:1775768269.489799:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775768269.489803:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5521 00000400:00000200:0.0:1775768269.489808:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768269.489811:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768269.489815:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775768269.489817:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000400:00000200:0.0:1775768269.489819:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000040:2.0:1775768269.489823:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768269.489826:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1775768269.489828:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00000040:2.0:1775768269.489831:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768269.489834:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000200:0.0:1775768269.489837:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:2.0:1775768269.489838:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775768269.489840:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:2.0:1775768269.489842:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775768269.489844:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000400:00000200:0.0:1775768269.489845:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000020:00000001:2.0:1775768269.489848:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:2.0:1775768269.489852:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:2.0:1775768269.489855:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768269.489858:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768269.489862:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000200:0.0:1775768269.489864:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000100:00000001:2.0:1775768269.489864:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775768269.489868:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000400:00000200:0.0:1775768269.489871:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00080000:2.0:1775768269.489874:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775768269.489877:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768269.489882:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225478:225478:256:4294967295] 192.168.202.156@tcp LPNI seq info [225474:225474:8:4294967295] 02000000:00000001:2.0:1775768269.489882:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775768269.489886:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768269.489890:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768269.489891:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768269.489894:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775768269.489896:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775768269.489898:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768269.489900:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768269.489906:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000100:00000010:2.0:1775768269.489907:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfed80. 00000020:00000040:2.0:1775768269.489911:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:2.0:1775768269.489915:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775768269.489916:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768269.489917:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775768269.489920:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:0.0:1775768269.489922:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 02000000:00000001:2.0:1775768269.489923:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775768269.489928:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1d400. 00000800:00000010:0.0:1775768269.489930:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6500. 00000100:00000001:2.0:1775768269.489932:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768269.489939:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfed80 x1862022294295296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:0.0:1775768269.489940:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768269.489947:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000100:00000001:2.0:1775768269.489956:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775768269.489959:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768269.489963:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb9b9eae] to pc [ptlrpcd_rcv+-1] req@ffff994207dfed80 x1862022294295296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768269.489970:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768269.489976:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:0.1:1775768269.489976:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768269.489982:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000100:00080000:2.0:1775768269.489983:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1775768269.489986:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775768269.489989:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1775768269.489993:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000100:00000001:2.0:1775768269.489994:0:36:0:(import.c:484:import_select_connection()) Process entered 00000800:00000010:0.0:1775768269.489997:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6500. 00000100:00080000:2.0:1775768269.489999:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5521 00000400:00000200:2.0:1775768269.490005:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768269.490006:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768269.490010:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775768269.490014:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000400:00000200:0.0:1775768269.490015:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000100:00000040:2.0:1775768269.490019:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768269.490023:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1775768269.490024:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000100:00000040:2.0:1775768269.490026:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000010:0.0:1775768269.490028:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:2.0:1775768269.490030:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:0.0:1775768269.490033:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:2.0:1775768269.490034:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775768269.490037:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:2.0:1775768269.490039:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000200:0.0:1775768269.490041:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfce00 x1862022294295040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768294 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1775768269.490042:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:2.0:1775768269.490046:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:2.0:1775768269.490051:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:2.0:1775768269.490058:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768269.490060:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1775768269.490061:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768269.490064:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000040:0.0:1775768269.490067:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfce00 x1862022294295040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768294 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768269.490068:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775768269.490072:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:2.0:1775768269.490079:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768269.490085:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768269.490085:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768269.490090:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1775768269.490091:0:2399:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:2.0:1775768269.490095:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768269.490097:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768269.490098:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768269.490101:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775768269.490104:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:2.0:1775768269.490106:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768269.490109:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768269.490112:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000010:2.0:1775768269.490113:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfe300. 00000100:00000001:0.0:1775768269.490115:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4300195645 00000020:00000040:2.0:1775768269.490117:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:2.0:1775768269.490121:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775768269.490122:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768269.490123:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942078a3d80. 00000100:00000001:2.0:1775768269.490124:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775768269.490127:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775768269.490130:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768269.490133:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 02000000:00000010:2.0:1775768269.490135:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1e800. 00000100:00000001:0.0:1775768269.490138:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4300195645 00000100:00000001:2.0:1775768269.490139:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768269.490145:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942078a3420. 00000100:00100000:2.0:1775768269.490147:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfe300 x1862022294295424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768269.490150:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775768269.490180:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768269.490181:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768269.490185:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768269.490185:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768269.490191:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000040:2.0:1775768269.490194:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c661fef] to pc [ptlrpcd_rcv+-1] req@ffff994207dfe300 x1862022294295424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768269.490198:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768269/real 1775768269] req@ffff994207dfce00 x1862022294295040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768269.490213:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1775768269.490216:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:2.0:1775768269.490217:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000001:0.0:1775768269.490220:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768269.490223:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbc268. 00000400:00000200:0.0:1775768269.490226:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014ee0 00000400:00000010:0.0:1775768269.490227:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014ee0. 00000100:00080000:2.0:1775768269.490231:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775768269.490232:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1775768269.490233:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775768269.490236:0:36:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:0.0:1775768269.490238:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfce00 x1862022294295040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768269.490240:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775768269.490246:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5521 00000400:00000200:2.0:1775768269.490254:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000200:0.0:1775768269.490257:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfce00 x1862022294295040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1775768269.490264:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775768269.490269:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1775768269.490270:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768269.490273:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768269.490274:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768269.490277:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768269.490279:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1775768269.490280:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768269.490283:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768269.490297:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775768269.490300:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775768269.490301:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:2.0:1775768269.490303:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775768269.490305:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:2.0:1775768269.490307:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00100000:0.0:1775768269.490310:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfce00 x1862022294295040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1775768269.490310:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:2.0:1775768269.490317:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768269.490321:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768269.490325:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768269.490328:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768269.490329:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775768269.490333:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00000040:0.0:1775768269.490338:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfce00 x1862022294295040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768294 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768269.490339:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775768269.490344:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768269.490349:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1775768269.490353:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1775768269.490353:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768269.490357:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768269.490358:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768269.490360:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775768269.490361:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768269.490364:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:2.0:1775768269.490364:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000100:00000001:0.0:1775768269.490367:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000008:00000001:2.0:1775768269.490369:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768269.490371:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768269.490371:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768269.490376:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000010:2.0:1775768269.490378:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfc700. 00000100:00080000:0.0:1775768269.490382:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000020:00000040:2.0:1775768269.490382:0:36:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:2.0:1775768269.490387:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1775768269.490388:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:2.0:1775768269.490390:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768269.490393:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000040:0.0:1775768269.490394:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfce00 x1862022294295040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768294 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1775768269.490395:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775768269.490398:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775768269.490405:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1d000. 00000100:00100000:0.0:1775768269.490409:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294295040:192.168.202.156@tcp:250:kworker.0 00000100:00000001:2.0:1775768269.490410:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768269.490417:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768269.490420:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfc700 x1862022294295552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768269.490421:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768269.490427:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfce00 x1862022294295040/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768294 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768269.490450:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1775768269.490452:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:0.0:1775768269.490453:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1775768269.490455:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768269.490457:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002bc00. 02000000:00000001:0.0:1775768269.490462:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000040:2.0:1775768269.490462:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a176dd1] to pc [ptlrpcd_rcv+-1] req@ffff994207dfc700 x1862022294295552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775768269.490465:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768269.490468:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:0.0:1775768269.490472:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768269.490477:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1e400. 00000100:00000001:2.0:1775768269.490478:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768269.490483:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000040:2.0:1775768269.490485:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5532) 02000000:00000001:0.0:1775768269.490486:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768269.490489:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfce00. 00000100:00000001:0.0:1775768269.490493:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768269.490494:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768269.490496:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768269.490503:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000080:00000001:2.0:1775768269.490504:0:36:0:(rw.c:1481:ll_writepages()) Process entered 00000100:00000001:0.0:1775768269.490505:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775768269.490507:0:36:0:(file.c:5250:cl_sync_file_range()) Process entered 00000100:00000001:0.0:1775768269.490508:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775768269.490512:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1775768269.490512:0:36:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00000001:0.0:1775768269.490514:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000010:2.0:1775768269.490518:0:36:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994210b32630. 00000100:00000040:0.0:1775768269.490520:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dff480 x1862022294295168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1775768269.490523:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994209d2cc00. 00000020:00000010:2.0:1775768269.490530:0:36:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994209d2d600. 00000100:00000001:0.0:1775768269.490531:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768269.490534:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1775768269.490535:0:36:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942182d6ba0. 02000000:00000001:0.0:1775768269.490537:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768269.490540:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768269.490542:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:2.0:1775768269.490542:0:36:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207ccc2f0. 00000100:00100000:0.0:1775768269.490548:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294295168:192.168.202.156@tcp:38:kworker.0 00020000:00000010:2.0:1775768269.490549:0:36:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994220156e40. 00000100:00000001:0.0:1775768269.490554:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768269.490557:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00008000:00000010:2.0:1775768269.490557:0:36:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942077a8e20. 02000000:00000001:0.0:1775768269.490559:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768269.490561:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000080:00000010:2.0:1775768269.490563:0:36:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff9942087472a0. 02000000:00000010:0.0:1775768269.490565:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002bc00. 02000000:00000001:0.0:1775768269.490568:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:2.0:1775768269.490569:0:36:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077b8000. 00000400:00000200:0.0:1775768269.490573:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000010:2.0:1775768269.490574:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994209d2d000. 00000400:00000010:0.0:1775768269.490577:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbc268. 00000008:00000010:2.0:1775768269.490579:0:36:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942077cd600. 00000400:00000010:0.0:1775768269.490583:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014ee0. 00020000:00000010:2.0:1775768269.490585:0:36:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff99420876d680. 00000100:00000200:0.0:1775768269.490587:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294295168, portal 10 00000100:00000001:0.0:1775768269.490590:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00008000:00000010:2.0:1775768269.490590:0:36:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076cf60. 00000100:00000001:0.0:1775768269.490593:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611849856 : -112966097701760 : ffff994207dff480) 00000080:00000010:2.0:1775768269.490596:0:36:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff994205066870. 00000100:00000040:0.0:1775768269.490600:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dff480 x1862022294295168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768294 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1775768269.490600:0:36:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107759908408 : -112965949643208 : ffff994210b32638) 00000020:00001000:2.0:1775768269.490604:0:36:0:(cl_object.c:831:cl_env_get()) 1@ffff994210b32630 00000020:00000001:2.0:1775768269.490610:0:36:0:(cl_object.c:151:cl_object_top()) ffff99420791a5f8 -> ffff99420791a5f8 00000100:00000001:0.0:1775768269.490612:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:2.0:1775768269.490613:0:36:0:(cl_io.c:125:__cl_io_init()) Process entered 00000100:00000040:0.0:1775768269.490617:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000080:00000001:2.0:1775768269.490617:0:36:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000400:00000010:0.0:1775768269.490621:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000080:00200000:2.0:1775768269.490621:0:36:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x10176:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000100:00000200:0.0:1775768269.490625:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294295168, offset 0 00000020:00000001:2.0:1775768269.490627:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775768269.490630:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000400:00000200:0.0:1775768269.490632:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000080:00000001:2.0:1775768269.490633:0:36:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768269.490638:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000002:2.0:1775768269.490639:0:36:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x10176:0x0]io ffff9942077b8060 type 6 ignore/verify layout 1/0 00000400:00000200:0.0:1775768269.490644:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00020000:00000002:2.0:1775768269.490644:0:36:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207dfb6c8) owner 0000000000000000/ffff994201348000 00000400:00000200:0.0:1775768269.490649:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00020000:00000001:2.0:1775768269.490650:0:36:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00000400:00000200:0.0:1775768269.490653:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00020000:00000001:2.0:1775768269.490653:0:36:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:2.0:1775768269.490657:0:36:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00000400:00000200:0.0:1775768269.490658:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00020000:00000001:2.0:1775768269.490660:0:36:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768269.490662:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00020000:00000001:2.0:1775768269.490664:0:36:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1775768269.490669:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225479:225479:256:4294967295] 192.168.202.156@tcp LPNI seq info [225475:225475:8:4294967295] 00020000:00000001:2.0:1775768269.490669:0:36:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:2.0:1775768269.490672:0:36:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768269.490676:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000400:00000200:0.0:1775768269.490677:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:2.0:1775768269.490678:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:2.0:1775768269.490680:0:36:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00020000:00000002:2.0:1775768269.490682:0:36:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207dfb6c8) owner 0000000000000000/ffff994201348000 00000020:00000001:2.0:1775768269.490687:0:36:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768269.490690:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000020:00000001:2.0:1775768269.490691:0:36:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:2.0:1775768269.490695:0:36:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000800:00000200:0.0:1775768269.490698:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00020000:00000001:2.0:1775768269.490699:0:36:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00020000:00200000:2.0:1775768269.490703:0:36:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000800:00000010:0.0:1775768269.490704:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6500. 00020000:00000002:2.0:1775768269.490707:0:36:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00000800:00000200:0.0:1775768269.490711:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00020000:00000001:2.0:1775768269.490713:0:36:0:(lov_io.c:155:lov_sub_get()) Process entered 00000800:00000200:0.0:1775768269.490715:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00020000:00000001:2.0:1775768269.490716:0:36:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000020:00000001:2.0:1775768269.490718:0:36:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775768269.490722:0:36:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994210b32090. 00000800:00000200:0.1:1775768269.490723:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768269.490728:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000020:00000010:2.0:1775768269.490729:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9942060d5a00. 00000800:00000100:0.0:1775768269.490734:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000020:00000010:2.0:1775768269.490734:0:36:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff9942060d5200. 00000800:00000010:0.0:1775768269.490738:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6500. 00000020:00000010:2.0:1775768269.490739:0:36:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff9942182d6de0. 00000400:00000200:0.0:1775768269.490742:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000008:00000010:2.0:1775768269.490744:0:36:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207cc8b28. 00000400:00000200:0.0:1775768269.490750:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00020000:00000010:2.0:1775768269.490750:0:36:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994220156840. 00000400:00000200:0.0:1775768269.490753:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00008000:00000010:2.0:1775768269.490755:0:36:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942077a9530. 00000400:00000010:0.0:1775768269.490756:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768269.490760:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000080:00000010:2.0:1775768269.490760:0:36:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff994208747738. 00000100:00000200:0.0:1775768269.490764:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dff480 x1862022294295168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768294 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000010:2.0:1775768269.490765:0:36:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077ba910. 00000020:00000010:2.0:1775768269.490770:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff9942060d4800. 00000008:00000010:2.0:1775768269.490776:0:36:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942077cd8c0. 00020000:00000010:2.0:1775768269.490780:0:36:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff99420876e400. 00008000:00000010:2.0:1775768269.490786:0:36:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff99421076c3f0. 00000080:00000010:2.0:1775768269.490791:0:36:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942050663c0. 00000100:00000001:0.0:1775768269.490792:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000020:00000001:2.0:1775768269.490796:0:36:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107759906968 : -112965949644648 : ffff994210b32098) 00000100:00000040:0.0:1775768269.490797:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dff480 x1862022294295168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768294 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00001000:2.0:1775768269.490800:0:36:0:(cl_object.c:831:cl_env_get()) 1@ffff994210b32090 00000020:00000001:2.0:1775768269.490805:0:36:0:(cl_object.c:151:cl_object_top()) ffff99420791ae90 -> ffff99420791a5f8 00000100:00000001:0.0:1775768269.490808:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768269.490809:0:36:0:(cl_io.c:125:__cl_io_init()) Process entered 00000100:00000001:0.0:1775768269.490812:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:2.0:1775768269.490813:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775768269.490815:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00000001:0.0:1775768269.490816:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768269.490817:0:36:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775768269.490820:0:36:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768269.490821:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00020000:00000001:2.0:1775768269.490822:0:36:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107621738216 : -112966087813400 : ffff99420876d6e8) 00000400:00000200:0.0:1775768269.490827:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:2.0:1775768269.490827:0:36:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000020:00000001:2.0:1775768269.490830:0:36:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768269.490831:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300194323 00020000:00000001:2.0:1775768269.490834:0:36:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768269.490836:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300195645 00000020:00000001:2.0:1775768269.490838:0:36:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768269.490840:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300194323 00000020:00000001:2.0:1775768269.490842:0:36:0:(cl_io.c:314:cl_io_lock()) Process entered 00000100:00000001:0.0:1775768269.490843:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300195645 00020000:00000001:2.0:1775768269.490844:0:36:0:(lov_io.c:1108:lov_io_lock()) Process entered 00000100:00000001:0.0:1775768269.490847:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:2.0:1775768269.490847:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775768269.490850:0:36:0:(cl_io.c:314:cl_io_lock()) Process entered 00000100:00000001:0.0:1775768269.490852:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768269.490852:0:36:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000100:00000001:0.0:1775768269.490855:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000020:00000001:2.0:1775768269.490856:0:36:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768269.490859:0:36:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768269.490861:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768269/real 1775768269] req@ffff994207dff480 x1862022294295168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:2.0:1775768269.490862:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775768269.490865:0:36:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768269.490870:0:36:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:2.0:1775768269.490872:0:36:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768269.490875:0:36:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768269.490879:0:36:0:(cl_io.c:528:cl_io_start()) Process entered 00000400:00000001:0.0:1775768269.490880:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00020000:00000001:2.0:1775768269.490882:0:36:0:(lov_io.c:1114:lov_io_start()) Process entered 00000400:00000001:0.0:1775768269.490883:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00020000:00000001:2.0:1775768269.490884:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000400:00000010:0.0:1775768269.490886:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbc268. 00000020:00000001:2.0:1775768269.490886:0:36:0:(cl_io.c:528:cl_io_start()) Process entered 00000008:00000001:2.0:1775768269.490889:0:36:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000400:00000200:0.0:1775768269.490890:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014ee0 00000008:00000001:2.0:1775768269.490892:0:36:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000400:00000010:0.0:1775768269.490893:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014ee0. 00000100:00000001:0.0:1775768269.490897:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000008:00000001:2.0:1775768269.490899:0:36:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000100:00000200:0.0:1775768269.490901:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dff480 x1862022294295168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000020:2.0:1775768269.490902:0:36:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:2.0:1775768269.490905:0:36:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1775768269.490910:0:36:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000008:00000001:2.0:1775768269.490913:0:36:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000008:00000001:2.0:1775768269.490916:0:36:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107754940544 : -112965954611072 : ffff994210675880) 00000100:00000200:0.0:1775768269.490917:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dff480 x1862022294295168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000020:2.0:1775768269.490922:0:36:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994210675880 ready 0|-|- wr 1|-|+ rd 0|- 10 in flight 00000008:00000001:2.0:1775768269.490929:0:36:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:2.0:1775768269.490931:0:36:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000100:00000001:0.0:1775768269.490932:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000008:00000001:2.0:1775768269.490934:0:36:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768269.490935:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1775768269.490936:0:36:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000100:00000001:0.0:1775768269.490938:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000008:00000020:2.0:1775768269.490940:0:36:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994218232fc0 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000100:00000001:0.0:1775768269.490941:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1775768269.490947:0:36:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768269.490948:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dff480 x1862022294295168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:2.0:1775768269.490951:0:36:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768269.490956:0:36:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775768269.490960:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768269.490962:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00020000:00000001:2.0:1775768269.490963:0:36:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768269.490966:0:36:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768269.490968:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dff480 x1862022294295168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768294 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1775768269.490969:0:36:0:(cl_io.c:555:cl_io_end()) Process entered 00020000:00000001:2.0:1775768269.490972:0:36:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00020000:00000001:2.0:1775768269.490975:0:36:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000020:00000001:2.0:1775768269.490977:0:36:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000001:2.0:1775768269.490980:0:36:0:(cl_io.c:563:cl_io_end()) Process leaving 00000100:00000001:0.0:1775768269.490982:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00020000:00000001:2.0:1775768269.490983:0:36:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768269.490988:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00020000:00000001:2.0:1775768269.490988:0:36:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000020:00000001:2.0:1775768269.490990:0:36:0:(cl_io.c:563:cl_io_end()) Process leaving 00000100:00000001:0.0:1775768269.490991:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:2.0:1775768269.490993:0:36:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000100:00000001:0.0:1775768269.490994:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768269.490997:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00020000:00000001:2.0:1775768269.490997:0:36:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00020000:00000001:2.0:1775768269.491000:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000001:0.0:1775768269.491002:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:2.0:1775768269.491003:0:36:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000100:00080000:0.0:1775768269.491006:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000020:00000001:2.0:1775768269.491006:0:36:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:2.0:1775768269.491008:0:36:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775768269.491012:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768269.491013:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00020000:00000001:2.0:1775768269.491014:0:36:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000100:00000001:0.0:1775768269.491017:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000001:2.0:1775768269.491018:0:36:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:2.0:1775768269.491021:0:36:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000020:00000001:2.0:1775768269.491023:0:36:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000100:00000040:0.0:1775768269.491024:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dff480 x1862022294295168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768294 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1775768269.491025:0:36:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:2.0:1775768269.491028:0:36:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00020000:00000001:2.0:1775768269.491030:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775768269.491033:0:36:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000001:2.0:1775768269.491035:0:36:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00020000:00000001:2.0:1775768269.491038:0:36:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768269.491041:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294295168:192.168.202.156@tcp:38:kworker.0 00020000:00000001:2.0:1775768269.491042:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775768269.491046:0:36:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000100:00000001:0.0:1775768269.491048:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000001:2.0:1775768269.491048:0:36:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000020:00000001:2.0:1775768269.491050:0:36:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768269.491054:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfed80 x1862022294295296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1775768269.491054:0:36:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:2.0:1775768269.491057:0:36:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:2.0:1775768269.491060:0:36:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:2.0:1775768269.491063:0:36:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:2.0:1775768269.491066:0:36:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000100:00000001:0.0:1775768269.491067:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00001000:2.0:1775768269.491069:0:36:0:(cl_object.c:925:cl_env_put()) 1@ffff994210b32090 00000100:00000001:0.0:1775768269.491070:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768269.491073:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000010:2.0:1775768269.491074:0:36:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff9942060d5200. 02000000:00000001:0.0:1775768269.491076:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768269.491079:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1775768269.491080:0:36:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942182d6de0. 00000100:00100000:0.0:1775768269.491085:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294295296:192.168.202.156@tcp:38:kworker.0 00000008:00000010:2.0:1775768269.491085:0:36:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207cc8b28. 00000100:00000001:0.0:1775768269.491091:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00020000:00000010:2.0:1775768269.491091:0:36:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994220156840. 02000000:00000001:0.0:1775768269.491093:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768269.491096:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:2.0:1775768269.491096:0:36:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942077a9530. 02000000:00000001:0.0:1775768269.491100:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000080:00000010:2.0:1775768269.491102:0:36:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff994208747738. 02000000:00000010:0.0:1775768269.491104:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220029800. 00000080:00000010:2.0:1775768269.491106:0:36:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077ba910. 02000000:00000001:0.0:1775768269.491107:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1775768269.491111:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9942060d5a00. 00000400:00000200:0.0:1775768269.491113:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00000010:2.0:1775768269.491116:0:36:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942077cd8c0. 00000400:00000010:0.0:1775768269.491117:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbc268. 00020000:00000010:2.0:1775768269.491122:0:36:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff99420876e400. 00000400:00000010:0.0:1775768269.491123:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014ee0. 00000100:00000200:0.0:1775768269.491129:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294295296, portal 10 00008000:00000010:2.0:1775768269.491129:0:36:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076c3f0. 00000100:00000001:0.0:1775768269.491132:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000080:00000010:2.0:1775768269.491134:0:36:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942050663c0. 00000100:00000001:0.0:1775768269.491135:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611848064 : -112966097703552 : ffff994207dfed80) 00000020:00000010:2.0:1775768269.491138:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff9942060d4800. 00000020:00000010:2.0:1775768269.491142:0:36:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994210b32090. 00000100:00000040:0.0:1775768269.491143:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfed80 x1862022294295296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768294 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:2.0:1775768269.491147:0:36:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:2.0:1775768269.491150:0:36:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:2.0:1775768269.491153:0:36:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000080:00200000:2.0:1775768269.491176:0:36:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x10176:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00000001:0.0:1775768269.491177:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768269.491183:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000080:00000001:2.0:1775768269.491183:0:36:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:2.0:1775768269.491186:0:36:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000400:00000010:0.0:1775768269.491188:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000020:00001000:2.0:1775768269.491189:0:36:0:(cl_object.c:925:cl_env_put()) 1@ffff994210b32630 00000100:00000200:0.0:1775768269.491191:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294295296, offset 0 00000020:00000010:2.0:1775768269.491192:0:36:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994209d2d600. 00000400:00000200:0.0:1775768269.491198:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000020:00000010:2.0:1775768269.491199:0:36:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff9942182d6ba0. 00000008:00000010:2.0:1775768269.491204:0:36:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207ccc2f0. 00000400:00000200:0.0:1775768269.491205:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000010:2.0:1775768269.491208:0:36:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994220156e40. 00000400:00000200:0.0:1775768269.491211:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00008000:00000010:2.0:1775768269.491213:0:36:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942077a8e20. 00000400:00000200:0.0:1775768269.491216:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000080:00000010:2.0:1775768269.491217:0:36:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff9942087472a0. 00000400:00000200:0.0:1775768269.491221:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000080:00000010:2.0:1775768269.491222:0:36:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077b8000. 00000400:00000200:0.0:1775768269.491226:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000020:00000010:2.0:1775768269.491226:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994209d2cc00. 00000400:00000200:0.0:1775768269.491231:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000008:00000010:2.0:1775768269.491232:0:36:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942077cd600. 00000400:00000200:0.0:1775768269.491236:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225480:225480:256:4294967295] 192.168.202.156@tcp LPNI seq info [225476:225476:8:4294967295] 00020000:00000010:2.0:1775768269.491238:0:36:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff99420876d680. 00008000:00000010:2.0:1775768269.491243:0:36:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff99421076cf60. 00000400:00000200:0.0:1775768269.491244:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000080:00000010:2.0:1775768269.491248:0:36:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff994205066870. 00000020:00000010:2.0:1775768269.491253:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994209d2d000. 00000400:00000200:0.0:1775768269.491257:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000020:00000010:2.0:1775768269.491258:0:36:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994210b32630. 00000080:00000001:2.0:1775768269.491263:0:36:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1775768269.491267:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000080:00000001:2.0:1775768269.491268:0:36:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1775768269.491271:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6500. 00000800:00000200:0.0:1775768269.491277:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768269.491280:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775768269.491285:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768269.491299:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1775768269.491304:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768269.491306:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6500. 00000400:00000200:0.0:1775768269.491309:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768269.491312:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768269.491315:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768269.491317:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768269.491319:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768269.491322:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfed80 x1862022294295296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768294 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768269.491330:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768269.491332:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfed80 x1862022294295296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768294 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768269.491339:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768269.491341:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768269.491345:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768269.491348:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:0.0:1775768269.491351:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768269.491353:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300194316 00000100:00000001:0.0:1775768269.491356:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300195645 00000100:00000001:0.0:1775768269.491358:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300194316 00000100:00000001:0.0:1775768269.491361:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300195645 00000100:00000001:0.0:1775768269.491363:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768269.491365:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768269.491367:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768269.491370:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768269/real 1775768269] req@ffff994207dfed80 x1862022294295296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768269.491378:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768269.491380:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768269.491381:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbc268. 00000400:00000200:0.0:1775768269.491384:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014ee0 00000400:00000010:0.0:1775768269.491386:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014ee0. 00000100:00000001:0.0:1775768269.491388:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768269.491392:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfed80 x1862022294295296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768269.491400:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfed80 x1862022294295296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768269.491406:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768269.491407:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768269.491410:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768269.491411:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768269.491415:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfed80 x1862022294295296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768269.491422:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768269.491422:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768269.491426:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768269.491428:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768269.491430:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfed80 x1862022294295296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768294 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768269.491430:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768269.491434:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768269.491436:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768269.491440:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768269.491442:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768269.491446:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768269.491447:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768269.491450:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768269.491452:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768269.491453:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768269.491455:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768269.491458:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768269.491460:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768269.491463:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768269.491465:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768269.491469:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfed80 x1862022294295296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768294 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768269.491481:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294295296:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768269.491484:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768269.491488:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfe300 x1862022294295424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768269.491494:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768269.491495:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768269.491497:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768269.491499:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768269.491501:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768269.491504:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294295424:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768269.491507:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768269.491508:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768269.491510:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768269.491511:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768269.491514:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220029400. 02000000:00000001:0.0:1775768269.491516:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768269.491520:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768269.491522:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbc268. 00000400:00000010:0.0:1775768269.491525:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014ee0. 00000100:00000200:0.0:1775768269.491527:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294295424, portal 4 00000100:00000001:0.0:1775768269.491529:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768269.491530:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611845376 : -112966097706240 : ffff994207dfe300) 00000100:00000040:0.0:1775768269.491535:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfe300 x1862022294295424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768294 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768269.491541:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768269.491543:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768269.491545:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768269.491547:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294295424, offset 0 00000400:00000200:0.0:1775768269.491550:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768269.491554:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768269.491556:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768269.491559:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768269.491561:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768269.491563:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768269.491565:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768269.491568:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225481:225481:256:4294967295] 192.168.202.156@tcp LPNI seq info [225477:225477:8:4294967295] 00000400:00000200:0.0:1775768269.491571:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768269.491576:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768269.491580:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768269.491582:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6500. 00000800:00000200:0.0:1775768269.491585:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768269.491587:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775768269.491591:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768269.491593:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1775768269.491596:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768269.491598:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6500. 00000400:00000200:0.0:1775768269.491600:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768269.491604:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768269.491606:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768269.491608:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768269.491610:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768269.491613:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfe300 x1862022294295424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768294 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768269.491620:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768269.491623:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfe300 x1862022294295424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768294 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768269.491629:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768269.491630:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768269.491633:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768269.491636:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:0.0:1775768269.491639:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768269.491641:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300194339 00000100:00000001:0.0:1775768269.491643:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300195645 00000100:00000001:0.0:1775768269.491645:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300194339 00000100:00000001:0.0:1775768269.491646:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300195645 00000100:00000001:0.0:1775768269.491648:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768269.491650:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768269.491652:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768269.491655:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768269/real 1775768269] req@ffff994207dfe300 x1862022294295424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768269.491662:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768269.491663:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768269.491665:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbc268. 00000400:00000200:0.0:1775768269.491667:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014ee0 00000400:00000010:0.0:1775768269.491669:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014ee0. 00000100:00000001:0.0:1775768269.491670:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768269.491673:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfe300 x1862022294295424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768269.491681:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfe300 x1862022294295424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768269.491686:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768269.491687:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768269.491689:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768269.491690:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768269.491693:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfe300 x1862022294295424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768269.491698:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768269.491701:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfe300 x1862022294295424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768294 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768269.491706:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768269.491708:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768269.491710:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768269.491711:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768269.491712:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768269.491715:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768269.491717:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768269.491720:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768269.491722:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768269.491725:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfe300 x1862022294295424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768294 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768269.491732:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294295424:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768269.491736:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768269.491739:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfc700 x1862022294295552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768269.491745:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768269.491746:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768269.491748:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768269.491750:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768269.491752:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768269.491754:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294295552:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768269.491756:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768269.491757:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768269.491758:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768269.491759:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768269.491761:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002a800. 02000000:00000001:0.0:1775768269.491762:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768269.491764:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768269.491766:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbc268. 00000400:00000010:0.0:1775768269.491768:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014ee0. 00000100:00000200:0.0:1775768269.491770:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294295552, portal 4 00000100:00000001:0.0:1775768269.491771:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768269.491772:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611838208 : -112966097713408 : ffff994207dfc700) 00000100:00000040:0.0:1775768269.491776:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfc700 x1862022294295552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768294 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768269.491779:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768269.491781:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768269.491782:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768269.491783:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294295552, offset 0 00000400:00000200:0.0:1775768269.491785:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768269.491788:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768269.491790:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768269.491792:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768269.491793:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768269.491795:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768269.491796:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768269.491798:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225482:225482:256:4294967295] 192.168.202.156@tcp LPNI seq info [225478:225478:8:4294967295] 00000400:00000200:0.0:1775768269.491801:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768269.491804:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768269.491807:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768269.491809:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6500. 00000800:00000200:0.0:1775768269.491811:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768269.491812:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775768269.491815:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768269.491816:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1775768269.491818:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768269.491820:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6500. 00000400:00000200:0.0:1775768269.491821:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768269.491823:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768269.491825:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768269.491826:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768269.491827:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768269.491829:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfc700 x1862022294295552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768294 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768269.491837:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768269.491839:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfc700 x1862022294295552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768294 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768269.491843:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768269.491844:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768269.491845:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768269.491847:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:0.0:1775768269.491848:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768269.491849:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300194342 00000100:00000001:0.0:1775768269.491850:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300195645 00000100:00000001:0.0:1775768269.491852:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300194342 00000100:00000001:0.0:1775768269.491853:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300195645 00000100:00000001:0.0:1775768269.491854:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768269.491855:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768269.491857:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768269.491858:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768269/real 1775768269] req@ffff994207dfc700 x1862022294295552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768269.491862:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768269.491863:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768269.491864:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbc268. 00000400:00000200:0.0:1775768269.491865:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014ee0 00000400:00000010:0.0:1775768269.491865:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014ee0. 00000100:00000001:0.0:1775768269.491866:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768269.491868:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfc700 x1862022294295552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768269.491872:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfc700 x1862022294295552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768269.491877:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768269.491879:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768269.491880:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768269.491881:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768269.491883:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfc700 x1862022294295552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768269.491889:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768269.491892:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfc700 x1862022294295552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768294 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768269.491899:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768269.491901:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768269.491902:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768269.491903:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768269.491905:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768269.491908:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768269.491910:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768269.491913:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768269.491915:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768269.491919:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfc700 x1862022294295552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768294 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768269.491926:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294295552:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768269.491930:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768269.491932:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768269.491935:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dff480 x1862022294295168/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768294 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768269.491941:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768269.491943:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768269.491945:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002bc00. 02000000:00000001:0.0:1775768269.491947:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768269.491948:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768269.491949:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:0.0:1775768269.491951:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768269.491952:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1c800. 02000000:00000001:0.0:1775768269.491954:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768269.491955:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768269.491956:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dff480. 00000100:00000001:0.0:1775768269.491958:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768269.491959:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768269.491960:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768269.491963:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfed80 x1862022294295296/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768294 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768269.491969:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768269.491970:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768269.491972:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220029800. 02000000:00000001:0.0:1775768269.491973:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768269.491974:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768269.491975:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:0.0:1775768269.491977:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768269.491978:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1d400. 02000000:00000001:0.0:1775768269.491981:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768269.491984:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768269.491985:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfed80. 00000100:00000001:0.0:1775768269.491987:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768269.491988:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768269.491990:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768269.491993:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfe300 x1862022294295424/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768294 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768269.491999:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768269.492000:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768269.492001:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220029400. 02000000:00000001:0.0:1775768269.492003:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768269.492004:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768269.492005:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:0.0:1775768269.492007:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768269.492009:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1e800. 02000000:00000001:0.0:1775768269.492011:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768269.492012:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768269.492013:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfe300. 00000100:00000001:0.0:1775768269.492015:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768269.492016:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768269.492018:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768269.492020:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfc700 x1862022294295552/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768294 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768269.492026:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768269.492027:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768269.492028:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002a800. 02000000:00000001:0.0:1775768269.492031:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768269.492032:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768269.492033:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:0.0:1775768269.492035:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768269.492036:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1d000. 02000000:00000001:0.0:1775768269.492038:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768269.492039:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768269.492039:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfc700. 00000100:00000001:0.0:1775768269.492042:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768269.492043:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768269.492044:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768269.492048:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768269.492049:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768269.492051:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775768269.492052:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768269.553220:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768269.553225:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768269.553228:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768269.553229:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768269.553232:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768269.553233:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768269.553236:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768269.553237:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768269.553239:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768269.553240:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768269.553256:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768269.553258:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768269.553261:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768269.617232:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768269.617239:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768269.617248:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768269.617253:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768269.617256:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768269.617260:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768269.617263:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768269.617265:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768269.617268:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768270.513235:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768270.513243:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768270.513245:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768270.513251:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775768270.513254:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768270.513258:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768270.513261:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768270.513263:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768270.513266:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768270.513379:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768270.513383:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768270.513384:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768270.513388:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768270.513390:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768270.513392:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768270.513394:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768270.513396:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768270.513397:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768270.514194:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768270.514197:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768270.514198:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768270.514200:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768270.514202:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768270.514203:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768270.514205:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768270.514206:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768270.514207:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768270.577256:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768270.577260:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768270.577264:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768270.577265:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768270.577267:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768270.577268:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768270.577271:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768270.577272:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768270.577274:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768270.577275:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768270.577328:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768270.577331:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768270.577334:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768270.641271:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768270.641275:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768270.641296:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768270.641299:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768270.641301:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768270.641304:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768270.641306:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768270.641312:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768270.641314:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768270.833272:0:15036:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4300195645, at 4300195648 00000100:00000040:1.0:1775768270.833272:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4300195645, at 4300195648 00000100:00000010:0.0:1775768270.833299:0:15036:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3d80. 00000100:00000010:1.0:1775768270.833304:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3420. 00000100:00000001:2.0:1775768271.537252:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768271.537252:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768271.537258:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768271.537259:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768271.537260:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768271.537261:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768271.537264:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:2.0:1775768271.537266:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768271.537283:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768271.537287:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768271.537288:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768271.537290:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768271.537293:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768271.537296:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768271.537299:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768271.537299:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768271.537302:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768271.537304:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768271.537330:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768271.537332:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768271.537333:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768271.537335:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768271.537337:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768271.537339:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768271.537341:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768271.537341:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768271.537343:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768271.601295:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768271.601302:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768271.601304:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768271.601307:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768271.601312:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768271.601315:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768271.601317:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768271.601319:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768271.601323:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768271.601324:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768271.601328:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768271.601329:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768271.601332:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768271.601333:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768271.601337:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768271.601339:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768271.601362:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768271.601364:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768271.601367:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768271.665291:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768271.665297:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768271.665299:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768271.665305:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768271.665308:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768271.665312:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768271.665320:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768271.665322:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768271.665324:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768272.561244:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768272.561244:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768272.561250:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768272.561251:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768272.561252:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768272.561253:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768272.561270:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768272.561274:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1775768272.561277:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768272.561278:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768272.561280:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768272.561281:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768272.561282:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768272.561284:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768272.561284:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768272.561286:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768272.561287:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768272.561291:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775768272.561292:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768272.561292:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768272.561295:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768272.561295:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768272.561299:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768272.561299:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768272.561303:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768272.561306:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768272.561308:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768272.625253:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768272.625275:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768272.625282:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768272.625283:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768272.625288:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768272.625290:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768272.625293:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768272.625295:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768272.625298:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768272.625299:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768272.625353:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768272.625355:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768272.625357:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768272.689277:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768272.689282:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768272.689284:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768272.689288:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768272.689290:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768272.689293:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768272.689295:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768272.689297:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768272.689299:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768273.585243:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768273.585243:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768273.585243:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768273.585259:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768273.585260:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768273.585261:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768273.585262:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768273.585268:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:2.0:1775768273.585268:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775768273.585270:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768273.585272:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768273.585273:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768273.585273:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768273.585277:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768273.585277:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775768273.585278:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775768273.585283:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768273.585283:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768273.585286:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768273.585287:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768273.585291:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768273.585291:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768273.585294:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768273.585297:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768273.585298:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768273.585300:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768273.585304:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768273.649244:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768273.649258:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768273.649264:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768273.649265:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768273.649268:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768273.649269:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768273.649271:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768273.649272:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768273.649274:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768273.649275:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768273.649289:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768273.649292:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768273.649298:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768273.713269:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768273.713276:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768273.713278:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768273.713283:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768273.713286:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768273.713290:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768273.713293:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768273.713295:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768273.713298:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768274.609263:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768274.609263:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768274.609269:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768274.609269:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768274.609271:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768274.609272:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768274.609277:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:2.0:1775768274.609278:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768274.609282:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768274.609282:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768274.609287:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768274.609287:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768274.609292:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768274.609295:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768274.609297:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1775768274.609297:0:8:0:(rw.c:1481:ll_writepages()) Process entered 00000100:00000001:2.0:1775768274.609301:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000080:00000001:1.0:1775768274.609302:0:8:0:(file.c:5250:cl_sync_file_range()) Process entered 00000100:00000001:2.0:1775768274.609303:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1775768274.609305:0:8:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00000001:2.0:1775768274.609306:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1775768274.609313:0:8:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994206aba240. 00000020:00000010:1.0:1775768274.609319:0:8:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994201342e00. 00000020:00000010:1.0:1775768274.609325:0:8:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994201343200. 00000020:00000010:1.0:1775768274.609329:0:8:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207cba3c0. 00000008:00000010:1.0:1775768274.609335:0:8:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207e02ca0. 00020000:00000010:1.0:1775768274.609340:0:8:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff9942031b3480. 00008000:00000010:1.0:1775768274.609345:0:8:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942054ab170. 00000080:00000010:1.0:1775768274.609351:0:8:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99422030bf18. 00000080:00000010:1.0:1775768274.609355:0:8:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff994205751da8. 00000100:00080000:3.0:1775768274.609355:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768274.609362:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000020:00000010:1.0:1775768274.609363:0:8:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994208a8ec00. 00000100:00000001:3.0:1775768274.609365:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000008:00000010:1.0:1775768274.609367:0:8:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff994210709ce0. 00000100:00080000:3.0:1775768274.609369:0:36:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00020000:00000010:1.0:1775768274.609372:0:8:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994205743180. 00000100:00000001:3.0:1775768274.609376:0:36:0:(import.c:484:import_select_connection()) Process entered 00008000:00000010:1.0:1775768274.609377:0:8:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994219fec110. 00000100:00000001:2.0:1775768274.609378:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000080:00000010:1.0:1775768274.609381:0:8:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942057e2960. 00000100:00000001:2.0:1775768274.609381:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768274.609382:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768274.609385:0:8:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107591643720 : -112966117907896 : ffff994206aba248) 00000100:00100000:2.0:1775768274.609385:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00080000:3.0:1775768274.609385:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5527 00000100:00000001:2.0:1775768274.609388:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00001000:1.0:1775768274.609389:0:8:0:(cl_object.c:831:cl_env_get()) 1@ffff994206aba240 00000100:00000001:2.0:1775768274.609391:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768274.609393:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000001:1.0:1775768274.609394:0:8:0:(cl_object.c:151:cl_object_top()) ffff9942078a76c0 -> ffff9942078a76c0 00000100:00000001:2.0:1775768274.609395:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1775768274.609398:0:8:0:(cl_io.c:125:__cl_io_init()) Process entered 00000100:00000001:2.0:1775768274.609399:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775768274.609399:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000080:00000001:1.0:1775768274.609401:0:8:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:1.0:1775768274.609404:0:8:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:1.0:1775768274.609408:0:8:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000400:00000200:3.0:1775768274.609409:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000020:00000001:1.0:1775768274.609410:0:8:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:1.0:1775768274.609414:0:8:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768274.609415:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00020000:00000002:1.0:1775768274.609418:0:8:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017a:0x0]io ffff994205751e08 type 6 ignore/verify layout 1/0 00000100:00000040:3.0:1775768274.609420:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00020000:00000002:1.0:1775768274.609424:0:8:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207c1f9b0) owner 0000000000000000/ffff994201224f80 00000100:00000001:3.0:1775768274.609425:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00020000:00000001:1.0:1775768274.609430:0:8:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00000100:00000040:3.0:1775768274.609430:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00020000:00000001:1.0:1775768274.609433:0:8:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00000100:00000001:3.0:1775768274.609435:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00020000:00000001:1.0:1775768274.609436:0:8:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00020000:00000001:1.0:1775768274.609438:0:8:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775768274.609440:0:8:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1775768274.609441:0:36:0:(genops.c:811:class_conn2export()) Process entered 00020000:00000001:1.0:1775768274.609443:0:8:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:1.0:1775768274.609445:0:8:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775768274.609445:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:1.0:1775768274.609447:0:8:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:1.0:1775768274.609449:0:8:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:3.0:1775768274.609449:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00020000:00000001:1.0:1775768274.609450:0:8:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00020000:00000002:1.0:1775768274.609453:0:8:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207c1f9b0) owner 0000000000000000/ffff994201224f80 00000020:00000040:3.0:1775768274.609454:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:1.0:1775768274.609455:0:8:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768274.609459:0:8:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:3.0:1775768274.609459:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:1.0:1775768274.609461:0:8:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000020:00000001:3.0:1775768274.609463:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00020000:00000001:1.0:1775768274.609464:0:8:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00020000:00200000:1.0:1775768274.609467:0:8:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00000040:3.0:1775768274.609469:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00020000:00000002:1.0:1775768274.609471:0:8:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00000100:00000001:3.0:1775768274.609472:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00020000:00000001:1.0:1775768274.609474:0:8:0:(lov_io.c:155:lov_sub_get()) Process entered 00000100:00000040:3.0:1775768274.609476:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00020000:00000001:1.0:1775768274.609477:0:8:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000020:00000001:1.0:1775768274.609479:0:8:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00000001:3.0:1775768274.609481:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000010:1.0:1775768274.609482:0:8:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994206ababd0. 00000020:00000010:1.0:1775768274.609486:0:8:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994208a8e000. 00000020:00000040:3.0:1775768274.609486:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000020:00000010:1.0:1775768274.609491:0:8:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994208a8f600. 00000100:00080000:3.0:1775768274.609493:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000020:00000010:1.0:1775768274.609495:0:8:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207cba240. 00000100:00000001:3.0:1775768274.609496:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:1.0:1775768274.609500:0:8:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207e00000. 02000000:00000001:3.0:1775768274.609500:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00020000:00000010:1.0:1775768274.609505:0:8:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff9942031b30c0. 02000000:00000001:3.0:1775768274.609505:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00008000:00000010:1.0:1775768274.609509:0:8:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942054a8a98. 02000000:00000001:3.0:1775768274.609510:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768274.609514:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000080:00000010:1.0:1775768274.609515:0:8:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99422030b0a8. 00000100:00000001:3.0:1775768274.609516:0:36:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:1.0:1775768274.609518:0:8:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff994205753d98. 00000020:00000010:1.0:1775768274.609523:0:8:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff99420120a200. 00000100:00000010:3.0:1775768274.609524:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ab480. 00000008:00000010:1.0:1775768274.609528:0:8:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff994210709080. 00000020:00000040:3.0:1775768274.609528:0:36:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 00020000:00000010:1.0:1775768274.609533:0:8:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff9942057418c0. 02000000:00000001:3.0:1775768274.609534:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775768274.609537:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:1.0:1775768274.609538:0:8:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994219fec368. 00000100:00000001:3.0:1775768274.609541:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000080:00000010:1.0:1775768274.609542:0:8:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942057e24b0. 02000000:00000001:3.0:1775768274.609545:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000020:00000001:1.0:1775768274.609546:0:8:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107591646168 : -112966117905448 : ffff994206ababd8) 00000020:00001000:1.0:1775768274.609548:0:8:0:(cl_object.c:831:cl_env_get()) 1@ffff994206ababd0 00000020:00000001:1.0:1775768274.609550:0:8:0:(cl_object.c:151:cl_object_top()) ffff9942078a7788 -> ffff9942078a76c0 02000000:00000001:3.0:1775768274.609551:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768274.609552:0:8:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:1.0:1775768274.609555:0:8:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:1.0:1775768274.609557:0:8:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:1.0:1775768274.609558:0:8:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775768274.609559:0:8:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775768274.609560:0:8:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107571233256 : -112966138318360 : ffff9942057431e8) 00000020:00000001:1.0:1775768274.609562:0:8:0:(cl_io.c:399:cl_io_iter_init()) Process entered 02000000:00000010:3.0:1775768274.609562:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0c00. 00000020:00000001:1.0:1775768274.609565:0:8:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775768274.609567:0:8:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768274.609568:0:8:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768274.609570:0:8:0:(cl_io.c:314:cl_io_lock()) Process entered 00020000:00000001:1.0:1775768274.609571:0:8:0:(lov_io.c:1108:lov_io_lock()) Process entered 00000100:00000001:3.0:1775768274.609571:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775768274.609574:0:8:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:1.0:1775768274.609575:0:8:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:1.0:1775768274.609576:0:8:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:1.0:1775768274.609577:0:8:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768274.609578:0:8:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775768274.609581:0:8:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768274.609581:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ab480 x1862022294295680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00020000:00000001:1.0:1775768274.609583:0:8:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768274.609587:0:8:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:1.0:1775768274.609588:0:8:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768274.609592:0:8:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768274.609594:0:8:0:(cl_io.c:528:cl_io_start()) Process entered 00020000:00000001:1.0:1775768274.609597:0:8:0:(lov_io.c:1114:lov_io_start()) Process entered 00020000:00000001:1.0:1775768274.609598:0:8:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:1.0:1775768274.609599:0:8:0:(cl_io.c:528:cl_io_start()) Process entered 00000100:00000001:3.0:1775768274.609599:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000008:00000001:1.0:1775768274.609600:0:8:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000008:00000001:1.0:1775768274.609602:0:8:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000100:00000001:3.0:1775768274.609604:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1775768274.609608:0:8:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000100:00000040:3.0:1775768274.609609:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000067a0ba74] to pc [ptlrpcd_rcv+-1] req@ffff9942099ab480 x1862022294295680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000020:1.0:1775768274.609610:0:8:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:1.0:1775768274.609611:0:8:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:1.0:1775768274.609615:0:8:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000008:00000001:1.0:1775768274.609617:0:8:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000008:00000001:1.0:1775768274.609620:0:8:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107884691392 : -112965824860224 : ffff994218232fc0) 00000008:00000020:1.0:1775768274.609623:0:8:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994218232fc0 ready 0|-|- wr 1|-|+ rd 0|- 10 in flight 00000008:00000001:1.0:1775768274.609625:0:8:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:1.0:1775768274.609626:0:8:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:1.0:1775768274.609626:0:8:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:1.0:1775768274.609627:0:8:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000008:00000020:1.0:1775768274.609629:0:8:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994210675880 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000100:00000001:3.0:1775768274.609631:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1775768274.609633:0:8:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775768274.609634:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000008:00000001:1.0:1775768274.609636:0:8:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768274.609637:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775768274.609640:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000020:00000001:1.0:1775768274.609641:0:8:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768274.609642:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00020000:00000001:1.0:1775768274.609645:0:8:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768274.609646:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775768274.609649:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5527 00020000:00000001:1.0:1775768274.609650:0:8:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768274.609654:0:8:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768274.609655:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:3.0:1775768274.609656:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:1.0:1775768274.609658:0:8:0:(cl_io.c:555:cl_io_end()) Process entered 00020000:00000001:1.0:1775768274.609660:0:8:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00000100:00000001:2.0:1775768274.609660:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000400:00000200:3.0:1775768274.609662:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00020000:00000001:1.0:1775768274.609663:0:8:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000100:00000001:2.0:1775768274.609663:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00080000:3.0:1775768274.609664:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000020:00000001:1.0:1775768274.609665:0:8:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000001:1.0:1775768274.609668:0:8:0:(cl_io.c:563:cl_io_end()) Process leaving 00000100:00000040:3.0:1775768274.609670:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00020000:00000001:1.0:1775768274.609671:0:8:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768274.609671:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ab480 x1862022294295680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:1.0:1775768274.609675:0:8:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000100:00000001:3.0:1775768274.609675:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:1.0:1775768274.609678:0:8:0:(cl_io.c:563:cl_io_end()) Process leaving 00000100:00000040:3.0:1775768274.609679:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000001:1.0:1775768274.609680:0:8:0:(cl_io.c:352:cl_io_unlock()) Process entered 00020000:00000001:1.0:1775768274.609683:0:8:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00000100:00000001:3.0:1775768274.609684:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00020000:00000001:1.0:1775768274.609685:0:8:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000001:2.0:1775768274.609685:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000001:1.0:1775768274.609688:0:8:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000100:00000001:2.0:1775768274.609688:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768274.609688:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000001:1.0:1775768274.609690:0:8:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000040:3.0:1775768274.609691:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00020000:00000001:1.0:1775768274.609692:0:8:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768274.609693:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:3.0:1775768274.609694:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00020000:00000001:1.0:1775768274.609695:0:8:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768274.609696:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:3.0:1775768274.609696:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00020000:00000001:1.0:1775768274.609699:0:8:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 02000000:00000001:2.0:1775768274.609700:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768274.609700:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:1.0:1775768274.609701:0:8:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:1.0:1775768274.609704:0:8:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000020:00000001:3.0:1775768274.609704:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:1.0:1775768274.609706:0:8:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000100:00100000:2.0:1775768274.609706:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ab480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294295680:192.168.202.156@tcp:250:kworker.0 00000020:00000001:1.0:1775768274.609708:0:8:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000100:00000040:3.0:1775768274.609709:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00020000:00000001:1.0:1775768274.609710:0:8:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00000100:00000001:2.0:1775768274.609712:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00020000:00000001:1.0:1775768274.609713:0:8:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:1.0:1775768274.609714:0:8:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000100:00000001:3.0:1775768274.609714:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:1.0:1775768274.609716:0:8:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 02000000:00000001:2.0:1775768274.609716:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:3.0:1775768274.609717:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00020000:00000001:1.0:1775768274.609719:0:8:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768274.609720:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768274.609720:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00020000:00000001:1.0:1775768274.609722:0:8:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775768274.609725:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00020000:00000001:1.0:1775768274.609726:0:8:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 02000000:00000001:2.0:1775768274.609726:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:3.0:1775768274.609730:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 02000000:00000010:2.0:1775768274.609732:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f400. 00000100:00000001:3.0:1775768274.609732:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768274.609735:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775768274.609737:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768274.609740:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775768274.609742:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768274.609746:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775768274.609746:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775768274.609750:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000400:00000010:2.0:1775768274.609751:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000008:00000001:3.0:1775768274.609753:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768274.609756:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775768274.609760:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000010:3.0:1775768274.609760:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a9180. 00000020:00000040:3.0:1775768274.609761:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:3.0:1775768274.609765:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775768274.609766:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768274.609767:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775768274.609768:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000020:00000001:1.0:1775768274.609769:0:8:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000100:00000200:2.0:1775768274.609770:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294295680, portal 25 02000000:00000001:3.0:1775768274.609770:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768274.609772:0:8:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768274.609775:0:8:0:(cl_io.c:71:cl_io_fini()) Process entered 00000100:00000001:2.0:1775768274.609775:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00020000:00000001:1.0:1775768274.609777:0:8:0:(lov_io.c:675:lov_io_fini()) Process entered 00000100:00000001:2.0:1775768274.609777:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640865920 : -112966068685696 : ffff9942099ab480) 02000000:00000010:3.0:1775768274.609777:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0000. 00020000:00000001:1.0:1775768274.609778:0:8:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:1.0:1775768274.609781:0:8:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:1.0:1775768274.609783:0:8:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000100:00000001:3.0:1775768274.609783:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768274.609785:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ab480 x1862022294295680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768299 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00001000:1.0:1775768274.609786:0:8:0:(cl_object.c:925:cl_env_put()) 1@ffff994206ababd0 00000020:00000010:1.0:1775768274.609790:0:8:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994208a8f600. 00000100:00100000:3.0:1775768274.609790:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a9180 x1862022294295808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000010:1.0:1775768274.609797:0:8:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207cba240. 00000008:00000010:1.0:1775768274.609802:0:8:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207e00000. 00000100:00000001:2.0:1775768274.609804:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00020000:00000010:1.0:1775768274.609807:0:8:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff9942031b30c0. 00000100:00000040:2.0:1775768274.609809:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000100:00000001:3.0:1775768274.609811:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00008000:00000010:1.0:1775768274.609812:0:8:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942054a8a98. 00000400:00000010:2.0:1775768274.609812:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000001:3.0:1775768274.609813:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:1.0:1775768274.609818:0:8:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99422030b0a8. 00000100:00000200:2.0:1775768274.609818:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294295680, offset 0 00000100:00000040:3.0:1775768274.609820:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bdc2586] to pc [ptlrpcd_rcv+-1] req@ffff9942099a9180 x1862022294295808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000010:1.0:1775768274.609822:0:8:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff994205753d98. 00000020:00000010:1.0:1775768274.609828:0:8:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994208a8e000. 00000400:00000200:2.0:1775768274.609828:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000008:00000010:1.0:1775768274.609833:0:8:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff994210709080. 00000100:00000001:3.0:1775768274.609834:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768274.609838:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00020000:00000010:1.0:1775768274.609839:0:8:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff9942057418c0. 00000400:00000200:2.0:1775768274.609839:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00008000:00000010:1.0:1775768274.609844:0:8:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994219fec368. 00000100:00080000:3.0:1775768274.609844:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775768274.609847:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000080:00000010:1.0:1775768274.609848:0:8:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942057e24b0. 00000400:00000200:2.0:1775768274.609848:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00080000:3.0:1775768274.609850:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000020:00000010:1.0:1775768274.609852:0:8:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff99420120a200. 00000100:00000001:3.0:1775768274.609854:0:36:0:(import.c:484:import_select_connection()) Process entered 00000020:00000010:1.0:1775768274.609855:0:8:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994206ababd0. 00000400:00000200:2.0:1775768274.609855:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00020000:00000001:1.0:1775768274.609859:0:8:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00000400:00000200:2.0:1775768274.609861:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000100:00080000:3.0:1775768274.609861:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5527 00020000:00000001:1.0:1775768274.609862:0:8:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:1.0:1775768274.609864:0:8:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000080:00200000:1.0:1775768274.609867:0:8:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000400:00000200:2.0:1775768274.609867:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768274.609869:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000080:00000001:1.0:1775768274.609870:0:8:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:1.0:1775768274.609871:0:8:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:1.0:1775768274.609873:0:8:0:(cl_object.c:925:cl_env_put()) 1@ffff994206aba240 00000400:00000200:2.0:1775768274.609874:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225483:225483:256:4294967295] 192.168.202.156@tcp LPNI seq info [225479:225479:8:4294967295] 00000020:00000010:1.0:1775768274.609875:0:8:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994201343200. 00000020:00000010:1.0:1775768274.609878:0:8:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207cba3c0. 00000400:00000200:2.0:1775768274.609878:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00000010:1.0:1775768274.609880:0:8:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207e02ca0. 00020000:00000010:1.0:1775768274.609882:0:8:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff9942031b3480. 00008000:00000010:1.0:1775768274.609885:0:8:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942054ab170. 00000080:00000010:1.0:1775768274.609887:0:8:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99422030bf18. 00000400:00000200:2.0:1775768274.609889:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000080:00000010:1.0:1775768274.609890:0:8:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff994205751da8. 00000020:00000010:1.0:1775768274.609892:0:8:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994201342e00. 00000008:00000010:1.0:1775768274.609895:0:8:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff994210709ce0. 00000800:00000200:2.0:1775768274.609895:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00020000:00000010:1.0:1775768274.609897:0:8:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994205743180. 00008000:00000010:1.0:1775768274.609900:0:8:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994219fec110. 00000800:00000010:2.0:1775768274.609900:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000080:00000010:1.0:1775768274.609902:0:8:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942057e2960. 00000800:00000200:2.0:1775768274.609906:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000010:1.0:1775768274.609909:0:8:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994208a8ec00. 00000800:00000200:2.0:1775768274.609910:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000020:00000010:1.0:1775768274.609911:0:8:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994206aba240. 00000080:00000001:1.0:1775768274.609913:0:8:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.1:1775768274.609915:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000080:00000001:1.0:1775768274.609916:0:8:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775768274.609918:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000080:00000001:1.0:1775768274.609921:0:8:0:(rw.c:1481:ll_writepages()) Process entered 00000800:00000100:2.0:1775768274.609922:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000080:00000001:1.0:1775768274.609923:0:8:0:(file.c:5250:cl_sync_file_range()) Process entered 00000020:00000001:1.0:1775768274.609925:0:8:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000800:00000010:2.0:1775768274.609925:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:3.0:1775768274.609926:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000010:1.0:1775768274.609927:0:8:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994206aba240. 00000400:00000200:2.0:1775768274.609929:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000010:1.0:1775768274.609932:0:8:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff99420120a200. 00000400:00000200:3.0:1775768274.609934:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000400:00000200:2.0:1775768274.609935:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000020:00000010:1.0:1775768274.609936:0:8:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994201342e00. 00000100:00080000:3.0:1775768274.609937:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000020:00000010:1.0:1775768274.609939:0:8:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207cba3c0. 00000400:00000200:2.0:1775768274.609941:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000100:00000040:3.0:1775768274.609943:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000008:00000010:1.0:1775768274.609944:0:8:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207e02ca0. 00000400:00000010:2.0:1775768274.609944:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:3.0:1775768274.609946:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00020000:00000010:1.0:1775768274.609947:0:8:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff9942031b3480. 00000100:00000001:2.0:1775768274.609948:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:3.0:1775768274.609950:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00008000:00000010:1.0:1775768274.609951:0:8:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942054ab170. 00000100:00000001:3.0:1775768274.609952:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000200:2.0:1775768274.609955:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ab480 x1862022294295680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768299 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775768274.609955:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000080:00000010:1.0:1775768274.609956:0:8:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99422030bf18. 00000020:00000040:3.0:1775768274.609958:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000080:00000010:1.0:1775768274.609960:0:8:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff994205751da8. 00000020:00000001:3.0:1775768274.609960:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775768274.609962:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000010:1.0:1775768274.609965:0:8:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994201343200. 00000020:00000001:3.0:1775768274.609965:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:3.0:1775768274.609969:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000008:00000010:1.0:1775768274.609970:0:8:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff994210709ce0. 00020000:00000010:1.0:1775768274.609975:0:8:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994205743180. 00000100:00000040:3.0:1775768274.609976:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00008000:00000010:1.0:1775768274.609978:0:8:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994219fec110. 00000100:00000001:3.0:1775768274.609981:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000080:00000010:1.0:1775768274.609982:0:8:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942057e2960. 00000100:00000001:2.0:1775768274.609982:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775768274.609984:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000001:1.0:1775768274.609985:0:8:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107591643720 : -112966117907896 : ffff994206aba248) 00000100:00000001:3.0:1775768274.609987:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000040:2.0:1775768274.609988:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ab480 x1862022294295680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768299 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00001000:1.0:1775768274.609989:0:8:0:(cl_object.c:831:cl_env_get()) 1@ffff994206aba240 00000020:00000040:3.0:1775768274.609991:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000020:00000001:1.0:1775768274.609992:0:8:0:(cl_object.c:151:cl_object_top()) ffff9942078a7f58 -> ffff9942078a7f58 00000020:00000001:1.0:1775768274.609996:0:8:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:1.0:1775768274.609998:0:8:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000100:00080000:3.0:1775768274.609998:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775768274.610000:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000080:00200000:1.0:1775768274.610001:0:8:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017b:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000100:00000001:3.0:1775768274.610002:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768274.610003:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:1.0:1775768274.610005:0:8:0:(cl_io.c:898:cl_io_slice_add()) Process entered 02000000:00000001:3.0:1775768274.610006:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000020:00000001:1.0:1775768274.610007:0:8:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:1.0:1775768274.610009:0:8:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768274.610009:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768274.610010:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000002:1.0:1775768274.610012:0:8:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017b:0x0]io ffff994205751e08 type 6 ignore/verify layout 1/0 02000000:00000001:3.0:1775768274.610013:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768274.610015:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775768274.610018:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00020000:00000002:1.0:1775768274.610019:0:8:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207c1f000) owner 0000000000000000/ffff994201224f80 00000100:00000001:2.0:1775768274.610019:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4300200765 00000008:00100000:3.0:1775768274.610020:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00020000:00000001:1.0:1775768274.610022:0:8:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:1.0:1775768274.610024:0:8:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00000100:00000010:2.0:1775768274.610024:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d6ba0. 00000008:00000001:3.0:1775768274.610024:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775768274.610027:0:8:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00020000:00000001:1.0:1775768274.610028:0:8:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768274.610028:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775768274.610031:0:8:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768274.610032:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000010:3.0:1775768274.610032:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8700. 00020000:00000001:1.0:1775768274.610033:0:8:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:1.0:1775768274.610036:0:8:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775768274.610036:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000100:00000001:2.0:1775768274.610038:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4300200765 00000020:00000001:1.0:1775768274.610039:0:8:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:1.0:1775768274.610041:0:8:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 02000000:00000001:3.0:1775768274.610041:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000010:2.0:1775768274.610043:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d6de0. 00020000:00000001:1.0:1775768274.610044:0:8:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 02000000:00000001:3.0:1775768274.610044:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:1.0:1775768274.610045:0:8:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207c1f000) owner 0000000000000000/ffff994201224f80 00000100:00000001:2.0:1775768274.610047:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1775768274.610047:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000020:00000001:1.0:1775768274.610049:0:8:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768274.610049:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000020:00000001:1.0:1775768274.610052:0:8:0:(cl_io.c:823:cl_io_loop()) Process entered 00000100:00000001:2.0:1775768274.610052:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775768274.610053:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768274.610054:0:8:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000100:00000001:2.0:1775768274.610054:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775768274.610057:0:8:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00020000:00200000:1.0:1775768274.610058:0:8:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00000001:2.0:1775768274.610058:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00020000:00000002:1.0:1775768274.610060:0:8:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 02000000:00000010:3.0:1775768274.610060:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2800. 00000100:00100000:2.0:1775768274.610063:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768274/real 1775768274] req@ffff9942099ab480 x1862022294295680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:1.0:1775768274.610066:0:8:0:(lov_io.c:155:lov_sub_get()) Process entered 00000100:00000001:3.0:1775768274.610066:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775768274.610067:0:8:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000020:00000001:1.0:1775768274.610069:0:8:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:1.0:1775768274.610073:0:8:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994206ababd0. 00000100:00100000:3.0:1775768274.610076:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8700 x1862022294295936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000010:1.0:1775768274.610077:0:8:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994208a8ec00. 00000020:00000010:1.0:1775768274.610081:0:8:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994208a8e000. 00000020:00000010:1.0:1775768274.610085:0:8:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207cba240. 00000400:00000001:2.0:1775768274.610088:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000008:00000010:1.0:1775768274.610089:0:8:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207e00000. 00000400:00000001:2.0:1775768274.610091:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00020000:00000010:1.0:1775768274.610092:0:8:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff9942031b30c0. 00000400:00000010:2.0:1775768274.610094:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00008000:00000010:1.0:1775768274.610098:0:8:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942054a8a98. 00000400:00000200:2.0:1775768274.610098:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000100:00000001:3.0:1775768274.610098:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000080:00000010:1.0:1775768274.610102:0:8:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99422030b0a8. 00000400:00000010:2.0:1775768274.610102:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:3.0:1775768274.610103:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:1.0:1775768274.610105:0:8:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff994205753d98. 00000100:00000001:2.0:1775768274.610105:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000010:1.0:1775768274.610108:0:8:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994208a8f600. 00000100:00000040:3.0:1775768274.610109:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000faa25e3b] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8700 x1862022294295936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000010:1.0:1775768274.610110:0:8:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff994210709080. 00000100:00000200:2.0:1775768274.610110:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ab480 x1862022294295680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000010:1.0:1775768274.610114:0:8:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff9942057418c0. 00008000:00000010:1.0:1775768274.610118:0:8:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994219fec368. 00000080:00000010:1.0:1775768274.610122:0:8:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942057e24b0. 00000020:00000001:1.0:1775768274.610125:0:8:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107591646168 : -112966117905448 : ffff994206ababd8) 00000020:00001000:1.0:1775768274.610128:0:8:0:(cl_object.c:831:cl_env_get()) 1@ffff994206ababd0 00000020:00000001:1.0:1775768274.610131:0:8:0:(cl_object.c:151:cl_object_top()) ffff9942078a75f8 -> ffff9942078a7f58 00000020:00000001:1.0:1775768274.610134:0:8:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:1.0:1775768274.610137:0:8:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:1.0:1775768274.610138:0:8:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:1.0:1775768274.610141:0:8:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1775768274.610141:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ab480 x1862022294295680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768274.610141:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775768274.610145:0:8:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768274.610147:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00020000:00000001:1.0:1775768274.610148:0:8:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107571233256 : -112966138318360 : ffff9942057431e8) 00000020:00000001:1.0:1775768274.610151:0:8:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000020:00000001:1.0:1775768274.610153:0:8:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768274.610173:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775768274.610174:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775768274.610176:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775768274.610177:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768274.610179:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768274.610183:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775768274.610184:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775768274.610186:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00020000:00000001:1.0:1775768274.610187:0:8:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768274.610189:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5527 00000020:00000001:1.0:1775768274.610190:0:8:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768274.610194:0:8:0:(cl_io.c:314:cl_io_lock()) Process entered 00000100:00100000:2.0:1775768274.610194:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ab480 x1862022294295680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:1.0:1775768274.610196:0:8:0:(lov_io.c:1108:lov_io_lock()) Process entered 00000400:00000200:3.0:1775768274.610196:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:1.0:1775768274.610198:0:8:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:1.0:1775768274.610200:0:8:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:1.0:1775768274.610202:0:8:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000400:00000200:3.0:1775768274.610204:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000020:00000001:1.0:1775768274.610205:0:8:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768274.610205:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1775768274.610207:0:8:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768274.610209:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00020000:00000001:1.0:1775768274.610210:0:8:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775768274.610212:0:8:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768274.610212:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ab480 x1862022294295680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768299 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1775768274.610215:0:8:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:1.0:1775768274.610216:0:8:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768274.610218:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000001:1.0:1775768274.610219:0:8:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768274.610222:0:8:0:(cl_io.c:528:cl_io_start()) Process entered 00020000:00000001:1.0:1775768274.610225:0:8:0:(lov_io.c:1114:lov_io_start()) Process entered 00000100:00000001:2.0:1775768274.610225:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775768274.610225:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00020000:00000001:1.0:1775768274.610228:0:8:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:1.0:1775768274.610229:0:8:0:(cl_io.c:528:cl_io_start()) Process entered 00000100:00000001:2.0:1775768274.610229:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:1.0:1775768274.610231:0:8:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000100:00000001:2.0:1775768274.610231:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:3.0:1775768274.610231:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000008:00000001:1.0:1775768274.610233:0:8:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000100:00000001:2.0:1775768274.610233:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768274.610236:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000008:00000001:1.0:1775768274.610259:0:8:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:1.0:1775768274.610261:0:8:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:1.0:1775768274.610262:0:8:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:1.0:1775768274.610264:0:8:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000008:00000001:1.0:1775768274.610267:0:8:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000020:00000001:3.0:1775768274.610267:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000008:00000001:1.0:1775768274.610269:0:8:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107884688256 : -112965824863360 : ffff994218232380) 00000020:00000040:3.0:1775768274.610269:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:3.0:1775768274.610272:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000008:00000020:1.0:1775768274.610273:0:8:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994218232380 ready 0|-|- wr 1|-|+ rd 0|- 9 in flight 00000020:00000040:3.0:1775768274.610274:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000100:00000001:2.0:1775768274.610277:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000008:00000001:1.0:1775768274.610279:0:8:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000100:00000001:2.0:1775768274.610279:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:3.0:1775768274.610279:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000008:00000020:1.0:1775768274.610281:0:8:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000020:00000001:3.0:1775768274.610283:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000008:00000001:1.0:1775768274.610284:0:8:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1775768274.610284:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000008:00000001:1.0:1775768274.610287:0:8:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000008:00000020:1.0:1775768274.610290:0:8:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994210674fc0 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000100:00000040:3.0:1775768274.610290:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00080000:2.0:1775768274.610291:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000008:00000001:1.0:1775768274.610296:0:8:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768274.610296:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1775768274.610296:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000008:00000001:1.0:1775768274.610300:0:8:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768274.610302:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000001:1.0:1775768274.610303:0:8:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768274.610303:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ab480 x1862022294295680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768299 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768274.610305:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00020000:00000001:1.0:1775768274.610306:0:8:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775768274.610309:0:8:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775768274.610310:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000020:00000001:1.0:1775768274.610311:0:8:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768274.610315:0:8:0:(cl_io.c:555:cl_io_end()) Process entered 00020000:00000001:1.0:1775768274.610317:0:8:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00000100:00080000:3.0:1775768274.610317:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00020000:00000001:1.0:1775768274.610318:0:8:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000020:00000001:1.0:1775768274.610321:0:8:0:(cl_io.c:555:cl_io_end()) Process entered 00000100:00100000:2.0:1775768274.610321:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ab480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294295680:192.168.202.156@tcp:250:kworker.0 00000100:00000001:3.0:1775768274.610322:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768274.610324:0:8:0:(cl_io.c:563:cl_io_end()) Process leaving 02000000:00000001:3.0:1775768274.610326:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00020000:00000001:1.0:1775768274.610327:0:8:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768274.610328:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00020000:00000001:1.0:1775768274.610330:0:8:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000100:00000001:2.0:1775768274.610330:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1775768274.610330:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1775768274.610332:0:8:0:(cl_io.c:563:cl_io_end()) Process leaving 02000000:00000001:3.0:1775768274.610334:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768274.610335:0:8:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000100:00000040:2.0:1775768274.610335:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ab480 x1862022294295680/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768299 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:1.0:1775768274.610337:0:8:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00020000:00000001:1.0:1775768274.610338:0:8:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000001:3.0:1775768274.610338:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000020:00000001:1.0:1775768274.610341:0:8:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000008:00100000:3.0:1775768274.610341:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000020:00000001:1.0:1775768274.610343:0:8:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:1.0:1775768274.610344:0:8:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768274.610346:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 00000008:00000001:3.0:1775768274.610347:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775768274.610348:0:8:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768274.610349:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1775768274.610351:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775768274.610352:0:8:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 02000000:00000010:2.0:1775768274.610352:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f400. 00000020:00000001:1.0:1775768274.610354:0:8:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:1.0:1775768274.610356:0:8:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000020:00000001:1.0:1775768274.610357:0:8:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 02000000:00000001:2.0:1775768274.610357:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000010:3.0:1775768274.610357:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8e00. 00000020:00000001:1.0:1775768274.610358:0:8:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000001:2.0:1775768274.610358:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00020000:00000001:1.0:1775768274.610361:0:8:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00000020:00000040:2.0:1775768274.610361:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000040:3.0:1775768274.610361:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00020000:00000001:1.0:1775768274.610363:0:8:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:2.0:1775768274.610364:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 00000020:00000001:1.0:1775768274.610365:0:8:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 02000000:00000001:3.0:1775768274.610366:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000020:00000001:1.0:1775768274.610367:0:8:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 02000000:00000010:2.0:1775768274.610367:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b0c00. 00020000:00000001:1.0:1775768274.610369:0:8:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768274.610369:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768274.610371:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00020000:00000001:1.0:1775768274.610374:0:8:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768274.610374:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1775768274.610374:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775768274.610377:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00020000:00000001:1.0:1775768274.610378:0:8:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000100:00000010:2.0:1775768274.610378:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ab480. 00000020:00000001:1.0:1775768274.610381:0:8:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000020:00000001:1.0:1775768274.610382:0:8:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768274.610382:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 02000000:00000001:3.0:1775768274.610382:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768274.610384:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1775768274.610387:0:8:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:1.0:1775768274.610388:0:8:0:(lov_io.c:675:lov_io_fini()) Process entered 00000100:00000001:2.0:1775768274.610388:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00020000:00000001:1.0:1775768274.610390:0:8:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 02000000:00000010:3.0:1775768274.610390:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2c00. 00000020:00000001:1.0:1775768274.610392:0:8:0:(cl_io.c:71:cl_io_fini()) Process entered 00000100:00000001:2.0:1775768274.610392:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:1.0:1775768274.610394:0:8:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000100:00000001:2.0:1775768274.610394:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:1.0:1775768274.610395:0:8:0:(cl_object.c:925:cl_env_put()) 1@ffff994206ababd0 00000100:00000001:3.0:1775768274.610396:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768274.610397:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000010:1.0:1775768274.610398:0:8:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994208a8e000. 00000100:00000001:2.0:1775768274.610399:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768274.610401:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000010:1.0:1775768274.610402:0:8:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207cba240. 00000008:00000010:1.0:1775768274.610405:0:8:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207e00000. 00020000:00000010:1.0:1775768274.610407:0:8:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff9942031b30c0. 00000100:00000040:2.0:1775768274.610407:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a9180 x1862022294295808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768274.610407:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8e00 x1862022294296064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00008000:00000010:1.0:1775768274.610411:0:8:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942054a8a98. 00000080:00000010:1.0:1775768274.610415:0:8:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99422030b0a8. 00000080:00000010:1.0:1775768274.610419:0:8:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff994205753d98. 00000020:00000010:1.0:1775768274.610423:0:8:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994208a8ec00. 00000008:00000010:1.0:1775768274.610427:0:8:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff994210709080. 00000100:00000001:2.0:1775768274.610429:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00020000:00000010:1.0:1775768274.610430:0:8:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff9942057418c0. 00000100:00000001:2.0:1775768274.610431:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768274.610434:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1775768274.610434:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00008000:00000010:1.0:1775768274.610435:0:8:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994219fec368. 02000000:00000001:2.0:1775768274.610437:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000080:00000010:1.0:1775768274.610438:0:8:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942057e24b0. 00000100:00000001:3.0:1775768274.610438:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768274.610441:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1775768274.610443:0:8:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994208a8f600. 00000100:00100000:2.0:1775768274.610445:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294295808:192.168.202.156@tcp:38:kworker.0 00000020:00000010:1.0:1775768274.610447:0:8:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994206ababd0. 00000100:00000040:3.0:1775768274.610447:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000864a6866] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8e00 x1862022294296064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:1.0:1775768274.610450:0:8:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00000100:00000001:2.0:1775768274.610450:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00020000:00000001:1.0:1775768274.610453:0:8:0:(lov_io.c:694:lov_io_fini()) Process leaving 02000000:00000001:2.0:1775768274.610453:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000080:00000001:1.0:1775768274.610455:0:8:0:(vvp_io.c:280:vvp_io_fini()) Process entered 02000000:00000001:2.0:1775768274.610456:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000080:00200000:1.0:1775768274.610458:0:8:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 02000000:00000001:2.0:1775768274.610460:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000080:00000001:1.0:1775768274.610462:0:8:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000100:00000001:3.0:1775768274.610462:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775768274.610463:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f400. 00000020:00000001:1.0:1775768274.610464:0:8:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:1.0:1775768274.610466:0:8:0:(cl_object.c:925:cl_env_put()) 1@ffff994206aba240 02000000:00000001:2.0:1775768274.610466:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768274.610466:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000010:1.0:1775768274.610470:0:8:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994201342e00. 00000100:00080000:3.0:1775768274.610471:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000400:00000200:2.0:1775768274.610473:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000010:1.0:1775768274.610474:0:8:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207cba3c0. 00000100:00000001:3.0:1775768274.610474:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000400:00000010:2.0:1775768274.610477:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000100:00080000:3.0:1775768274.610477:0:36:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000008:00000010:1.0:1775768274.610478:0:8:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207e02ca0. 00020000:00000010:1.0:1775768274.610482:0:8:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff9942031b3480. 00000400:00000010:2.0:1775768274.610483:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000001:3.0:1775768274.610483:0:36:0:(import.c:484:import_select_connection()) Process entered 00008000:00000010:1.0:1775768274.610486:0:8:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942054ab170. 00000100:00080000:3.0:1775768274.610488:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5527 00000080:00000010:1.0:1775768274.610491:0:8:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99422030bf18. 00000100:00000200:2.0:1775768274.610494:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294295808, portal 10 00000400:00000200:3.0:1775768274.610494:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000080:00000010:1.0:1775768274.610496:0:8:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff994205751da8. 00000100:00000001:2.0:1775768274.610498:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768274.610499:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640856960 : -112966068694656 : ffff9942099a9180) 00000020:00000010:1.0:1775768274.610500:0:8:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff99420120a200. 00000400:00000200:3.0:1775768274.610501:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000008:00000010:1.0:1775768274.610503:0:8:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff994210709ce0. 00000100:00080000:3.0:1775768274.610504:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00020000:00000010:1.0:1775768274.610508:0:8:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994205743180. 00000100:00000040:2.0:1775768274.610508:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a9180 x1862022294295808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768299 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775768274.610509:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00008000:00000010:1.0:1775768274.610511:0:8:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994219fec110. 00000100:00000001:3.0:1775768274.610515:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000080:00000010:1.0:1775768274.610516:0:8:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942057e2960. 00000020:00000010:1.0:1775768274.610519:0:8:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994201343200. 00000100:00000040:3.0:1775768274.610520:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000010:1.0:1775768274.610522:0:8:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994206aba240. 00000100:00000001:2.0:1775768274.610522:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1775768274.610522:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000080:00000001:1.0:1775768274.610525:0:8:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1775768274.610526:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000080:00000001:1.0:1775768274.610527:0:8:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768274.610527:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000020:00000040:3.0:1775768274.610529:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000400:00000010:2.0:1775768274.610531:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000080:00000001:1.0:1775768274.610532:0:8:0:(rw.c:1481:ll_writepages()) Process entered 00000020:00000001:3.0:1775768274.610532:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000080:00000001:1.0:1775768274.610535:0:8:0:(file.c:5250:cl_sync_file_range()) Process entered 00000020:00000040:3.0:1775768274.610535:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000100:00000200:2.0:1775768274.610536:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294295808, offset 0 00000020:00000001:1.0:1775768274.610538:0:8:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000001:3.0:1775768274.610539:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000010:1.0:1775768274.610540:0:8:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994206aba240. 00000400:00000200:2.0:1775768274.610543:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000020:00000001:3.0:1775768274.610544:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000010:1.0:1775768274.610545:0:8:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994208a8f600. 00000020:00000010:1.0:1775768274.610549:0:8:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994208a8ec00. 00000400:00000200:2.0:1775768274.610549:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000040:3.0:1775768274.610551:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000010:1.0:1775768274.610552:0:8:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207cba3c0. 00000100:00000001:3.0:1775768274.610553:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1775768274.610556:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000008:00000010:1.0:1775768274.610557:0:8:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207e02ca0. 00000100:00000040:3.0:1775768274.610558:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00020000:00000010:1.0:1775768274.610561:0:8:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff9942031b3480. 00000400:00000200:2.0:1775768274.610561:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1775768274.610562:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00008000:00000010:1.0:1775768274.610564:0:8:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942054ab170. 00000020:00000040:3.0:1775768274.610564:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000400:00000200:2.0:1775768274.610566:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000080:00000010:1.0:1775768274.610568:0:8:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99422030bf18. 00000100:00080000:3.0:1775768274.610570:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000080:00000010:1.0:1775768274.610571:0:8:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff994205751da8. 00000400:00000200:2.0:1775768274.610572:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1775768274.610575:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1775768274.610576:0:8:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994208a8e000. 00000400:00000200:2.0:1775768274.610577:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 02000000:00000001:3.0:1775768274.610579:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000008:00000010:1.0:1775768274.610580:0:8:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff994210709ce0. 00020000:00000010:1.0:1775768274.610583:0:8:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994205743180. 00000400:00000200:2.0:1775768274.610583:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225484:225484:256:4294967295] 192.168.202.156@tcp LPNI seq info [225480:225480:8:4294967295] 02000000:00000001:3.0:1775768274.610583:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00008000:00000010:1.0:1775768274.610587:0:8:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994219fec110. 02000000:00000001:3.0:1775768274.610588:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:1.0:1775768274.610590:0:8:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942057e2960. 00000100:00000001:3.0:1775768274.610591:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:2.0:1775768274.610592:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:1.0:1775768274.610594:0:8:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107591643720 : -112966117907896 : ffff994206aba248) 00000008:00100000:3.0:1775768274.610594:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000020:00001000:1.0:1775768274.610598:0:8:0:(cl_object.c:831:cl_env_get()) 1@ffff994206aba240 00000008:00000001:3.0:1775768274.610599:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768274.610600:0:8:0:(cl_object.c:151:cl_object_top()) ffff99420791a850 -> ffff99420791a850 00000100:00000001:3.0:1775768274.610602:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768274.610603:0:8:0:(cl_io.c:125:__cl_io_init()) Process entered 00000400:00000200:2.0:1775768274.610603:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000080:00000001:1.0:1775768274.610605:0:8:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:1.0:1775768274.610607:0:8:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017f:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000100:00000010:3.0:1775768274.610608:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8380. 00000020:00000001:1.0:1775768274.610610:0:8:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000800:00000200:2.0:1775768274.610610:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000020:00000040:3.0:1775768274.610612:0:36:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:1.0:1775768274.610613:0:8:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000800:00000010:2.0:1775768274.610614:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000080:00000001:1.0:1775768274.610615:0:8:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768274.610617:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00020000:00000002:1.0:1775768274.610618:0:8:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017f:0x0]io ffff994205751e08 type 6 ignore/verify layout 1/0 02000000:00000001:3.0:1775768274.610619:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1775768274.610620:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00020000:00000002:1.0:1775768274.610624:0:8:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207dfbd90) owner 0000000000000000/ffff994201224f80 00000100:00000001:3.0:1775768274.610624:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000800:00000200:2.0:1775768274.610625:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 02000000:00000001:3.0:1775768274.610627:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00020000:00000001:1.0:1775768274.610628:0:8:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:1.0:1775768274.610630:0:8:0:(lov_io.c:495:lov_io_slice_init()) Process entered 02000000:00000001:3.0:1775768274.610630:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775768274.610631:0:8:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00020000:00000001:1.0:1775768274.610634:0:8:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775768274.610634:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00020000:00000001:1.0:1775768274.610637:0:8:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000010:3.0:1775768274.610637:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0c00. 00000800:00000200:2.1:1775768274.610637:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00020000:00000001:1.0:1775768274.610641:0:8:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00000100:00000001:3.0:1775768274.610642:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775768274.610643:0:8:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1775768274.610644:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000020:00000001:1.0:1775768274.610647:0:8:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000800:00000010:2.0:1775768274.610647:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000020:00000001:1.0:1775768274.610649:0:8:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000400:00000200:2.0:1775768274.610650:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00100000:3.0:1775768274.610650:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8380 x1862022294296192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00020000:00000001:1.0:1775768274.610651:0:8:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00020000:00000002:1.0:1775768274.610653:0:8:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207dfbd90) owner 0000000000000000/ffff994201224f80 00000020:00000001:1.0:1775768274.610658:0:8:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768274.610658:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000020:00000001:1.0:1775768274.610661:0:8:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:1.0:1775768274.610663:0:8:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000400:00000200:2.0:1775768274.610664:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00020000:00000001:1.0:1775768274.610666:0:8:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00000400:00000010:2.0:1775768274.610667:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00020000:00200000:1.0:1775768274.610668:0:8:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00000001:3.0:1775768274.610669:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775768274.610670:0:2399:0:(events.c:34:request_out_callback()) Process entered 00020000:00000002:1.0:1775768274.610672:0:8:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00000100:00000001:3.0:1775768274.610673:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1775768274.610675:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a9180 x1862022294295808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768299 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:1.0:1775768274.610678:0:8:0:(lov_io.c:155:lov_sub_get()) Process entered 00020000:00000001:1.0:1775768274.610681:0:8:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000100:00000040:3.0:1775768274.610681:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055f5621a] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8380 x1862022294296192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1775768274.610683:0:8:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:1.0:1775768274.610686:0:8:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff994206ababd0. 00000020:00000010:1.0:1775768274.610691:0:8:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff99420120a200. 00000100:00000001:2.0:1775768274.610694:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000020:00000010:1.0:1775768274.610695:0:8:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994201343200. 00000020:00000010:1.0:1775768274.610699:0:8:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207cba240. 00000100:00000040:2.0:1775768274.610700:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a9180 x1862022294295808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768299 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768274.610701:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:1.0:1775768274.610702:0:8:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994207e00000. 00020000:00000010:1.0:1775768274.610706:0:8:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff9942031b30c0. 00000100:00000040:3.0:1775768274.610707:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5537) 00008000:00000010:1.0:1775768274.610711:0:8:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff9942054a8a98. 00000100:00000001:2.0:1775768274.610712:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768274.610713:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000080:00000010:1.0:1775768274.610714:0:8:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99422030b0a8. 00000080:00000010:1.0:1775768274.610716:0:8:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff994205753d98. 00000100:00000001:2.0:1775768274.610716:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1775768274.610719:0:8:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994201342e00. 00000100:00000040:2.0:1775768274.610720:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000008:00000010:1.0:1775768274.610721:0:8:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff994210709080. 00020000:00000010:1.0:1775768274.610724:0:8:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff9942057418c0. 00000400:00000200:2.0:1775768274.610724:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00008000:00000010:1.0:1775768274.610726:0:8:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994219fec368. 00000100:00000001:2.0:1775768274.610726:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300199443 00000080:00000010:1.0:1775768274.610729:0:8:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942057e24b0. 00000100:00000001:2.0:1775768274.610729:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300200765 00000020:00000001:1.0:1775768274.610732:0:8:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107591646168 : -112966117905448 : ffff994206ababd8) 00000100:00000001:2.0:1775768274.610732:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300199443 00000020:00001000:1.0:1775768274.610734:0:8:0:(cl_object.c:831:cl_env_get()) 1@ffff994206ababd0 00000100:00000001:2.0:1775768274.610734:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300200765 00000020:00000001:1.0:1775768274.610736:0:8:0:(cl_object.c:151:cl_object_top()) ffff99420791a210 -> ffff99420791a850 00000100:00000001:2.0:1775768274.610736:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1775768274.610737:0:8:0:(cl_io.c:125:__cl_io_init()) Process entered 00000100:00000001:2.0:1775768274.610738:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768274.610739:0:8:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:1.0:1775768274.610740:0:8:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00000001:2.0:1775768274.610740:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000020:00000001:1.0:1775768274.610741:0:8:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775768274.610743:0:8:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768274.610743:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768274/real 1775768274] req@ffff9942099a9180 x1862022294295808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:1.0:1775768274.610745:0:8:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107571233256 : -112966138318360 : ffff9942057431e8) 00000020:00000001:1.0:1775768274.610747:0:8:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000020:00000001:1.0:1775768274.610748:0:8:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775768274.610750:0:8:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768274.610751:0:8:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1775768274.610751:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768274.610752:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:1.0:1775768274.610753:0:8:0:(cl_io.c:314:cl_io_lock()) Process entered 00020000:00000001:1.0:1775768274.610754:0:8:0:(lov_io.c:1108:lov_io_lock()) Process entered 00000400:00000010:2.0:1775768274.610754:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00020000:00000001:1.0:1775768274.610755:0:8:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:1.0:1775768274.610756:0:8:0:(cl_io.c:314:cl_io_lock()) Process entered 00000400:00000200:2.0:1775768274.610756:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000020:00000001:1.0:1775768274.610757:0:8:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:1.0:1775768274.610758:0:8:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775768274.610758:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000020:00000001:1.0:1775768274.610760:0:8:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768274.610760:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00020000:00000001:1.0:1775768274.610762:0:8:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1775768274.610762:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a9180 x1862022294295808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:1.0:1775768274.610763:0:8:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768274.610765:0:8:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:1.0:1775768274.610766:0:8:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768274.610768:0:8:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768274.610769:0:8:0:(cl_io.c:528:cl_io_start()) Process entered 00020000:00000001:1.0:1775768274.610770:0:8:0:(lov_io.c:1114:lov_io_start()) Process entered 00020000:00000001:1.0:1775768274.610771:0:8:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000200:2.0:1775768274.610771:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a9180 x1862022294295808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1775768274.610772:0:8:0:(cl_io.c:528:cl_io_start()) Process entered 00000008:00000001:1.0:1775768274.610773:0:8:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000008:00000001:1.0:1775768274.610774:0:8:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000008:00000001:1.0:1775768274.610776:0:8:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:1.0:1775768274.610777:0:8:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000100:00000001:2.0:1775768274.610777:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000008:00000001:1.0:1775768274.610778:0:8:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768274.610779:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:1.0:1775768274.610780:0:8:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000100:00000001:2.0:1775768274.610780:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000008:00000001:1.0:1775768274.610781:0:8:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000008:00000001:1.0:1775768274.610782:0:8:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107754938304 : -112965954613312 : ffff994210674fc0) 00000100:00000001:2.0:1775768274.610782:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00000020:1.0:1775768274.610785:0:8:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994210674fc0 ready 0|-|- wr 1|-|+ rd 0|- 9 in flight 00000100:00100000:2.0:1775768274.610785:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a9180 x1862022294295808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:1.0:1775768274.610788:0:8:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:1.0:1775768274.610789:0:8:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:1.0:1775768274.610790:0:8:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:1.0:1775768274.610792:0:8:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000100:00000001:2.0:1775768274.610792:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000008:00000020:1.0:1775768274.610794:0:8:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994218232380 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000100:00000040:2.0:1775768274.610795:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a9180 x1862022294295808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768299 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:1.0:1775768274.610797:0:8:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:1.0:1775768274.610799:0:8:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768274.610800:0:8:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775768274.610802:0:8:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768274.610802:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00020000:00000001:1.0:1775768274.610804:0:8:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768274.610805:0:8:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768274.610805:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768274.610806:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:1.0:1775768274.610807:0:8:0:(cl_io.c:555:cl_io_end()) Process entered 00000100:00000001:2.0:1775768274.610807:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00020000:00000001:1.0:1775768274.610808:0:8:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00020000:00000001:1.0:1775768274.610809:0:8:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000100:00000001:2.0:1775768274.610809:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000020:00000001:1.0:1775768274.610810:0:8:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000001:1.0:1775768274.610811:0:8:0:(cl_io.c:563:cl_io_end()) Process leaving 00000100:00000001:2.0:1775768274.610811:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00020000:00000001:1.0:1775768274.610812:0:8:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775768274.610813:0:8:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000100:00080000:2.0:1775768274.610813:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000020:00000001:1.0:1775768274.610814:0:8:0:(cl_io.c:563:cl_io_end()) Process leaving 00000020:00000001:1.0:1775768274.610815:0:8:0:(cl_io.c:352:cl_io_unlock()) Process entered 00020000:00000001:1.0:1775768274.610816:0:8:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00000100:00080000:2.0:1775768274.610816:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00020000:00000001:1.0:1775768274.610818:0:8:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000001:2.0:1775768274.610818:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000001:1.0:1775768274.610819:0:8:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000020:00000001:1.0:1775768274.610820:0:8:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:1.0:1775768274.610821:0:8:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768274.610822:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a9180 x1862022294295808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768299 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:1.0:1775768274.610823:0:8:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775768274.610825:0:8:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000020:00000001:1.0:1775768274.610826:0:8:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:1.0:1775768274.610827:0:8:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000020:00000001:1.0:1775768274.610828:0:8:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000001:1.0:1775768274.610829:0:8:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:1.0:1775768274.610830:0:8:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00000100:00100000:2.0:1775768274.610830:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294295808:192.168.202.156@tcp:38:kworker.0 00020000:00000001:1.0:1775768274.610831:0:8:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:1.0:1775768274.610832:0:8:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000001:1.0:1775768274.610833:0:8:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00020000:00000001:1.0:1775768274.610834:0:8:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768274.610834:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00020000:00000001:1.0:1775768274.610835:0:8:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1775768274.610837:0:8:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000100:00000040:2.0:1775768274.610837:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a8700 x1862022294295936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1775768274.610838:0:8:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000020:00000001:1.0:1775768274.610839:0:8:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768274.610840:0:8:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:1.0:1775768274.610841:0:8:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:1.0:1775768274.610842:0:8:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:1.0:1775768274.610843:0:8:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:1.0:1775768274.610844:0:8:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000100:00000001:2.0:1775768274.610844:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00001000:1.0:1775768274.610845:0:8:0:(cl_object.c:925:cl_env_put()) 1@ffff994206ababd0 00000100:00000001:2.0:1775768274.610845:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1775768274.610847:0:8:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994201343200. 02000000:00000001:2.0:1775768274.610847:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768274.610848:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1775768274.610849:0:8:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207cba240. 02000000:00000001:2.0:1775768274.610850:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:1.0:1775768274.610852:0:8:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207e00000. 00000100:00100000:2.0:1775768274.610853:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294295936:192.168.202.156@tcp:38:kworker.0 00020000:00000010:1.0:1775768274.610854:0:8:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff9942031b30c0. 00008000:00000010:1.0:1775768274.610856:0:8:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942054a8a98. 00000100:00000001:2.0:1775768274.610856:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000080:00000010:1.0:1775768274.610858:0:8:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99422030b0a8. 02000000:00000001:2.0:1775768274.610858:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768274.610859:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:1.0:1775768274.610860:0:8:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff994205753d98. 02000000:00000001:2.0:1775768274.610861:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000010:1.0:1775768274.610862:0:8:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff99420120a200. 02000000:00000010:2.0:1775768274.610864:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f000. 00000008:00000010:1.0:1775768274.610865:0:8:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff994210709080. 02000000:00000001:2.0:1775768274.610866:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:1.0:1775768274.610867:0:8:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff9942057418c0. 00008000:00000010:1.0:1775768274.610869:0:8:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994219fec368. 00000400:00000200:2.0:1775768274.610869:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000080:00000010:1.0:1775768274.610872:0:8:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942057e24b0. 00000400:00000010:2.0:1775768274.610872:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000020:00000010:1.0:1775768274.610874:0:8:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994201342e00. 00000400:00000010:2.0:1775768274.610875:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000020:00000010:1.0:1775768274.610876:0:8:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994206ababd0. 00000100:00000200:2.0:1775768274.610877:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294295936, portal 10 00020000:00000001:1.0:1775768274.610878:0:8:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:1.0:1775768274.610879:0:8:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000100:00000001:2.0:1775768274.610879:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000080:00000001:1.0:1775768274.610880:0:8:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000100:00000001:2.0:1775768274.610880:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640854272 : -112966068697344 : ffff9942099a8700) 00000080:00200000:1.0:1775768274.610882:0:8:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017f:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:1.0:1775768274.610884:0:8:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:1.0:1775768274.610885:0:8:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000100:00000040:2.0:1775768274.610885:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a8700 x1862022294295936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768299 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00001000:1.0:1775768274.610887:0:8:0:(cl_object.c:925:cl_env_put()) 1@ffff994206aba240 00000020:00000010:1.0:1775768274.610889:0:8:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994208a8ec00. 00000020:00000010:1.0:1775768274.610891:0:8:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207cba3c0. 00000100:00000001:2.0:1775768274.610891:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00000010:1.0:1775768274.610893:0:8:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994207e02ca0. 00000100:00000040:2.0:1775768274.610893:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00020000:00000010:1.0:1775768274.610895:0:8:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff9942031b3480. 00000400:00000010:2.0:1775768274.610895:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00008000:00000010:1.0:1775768274.610898:0:8:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff9942054ab170. 00000100:00000200:2.0:1775768274.610898:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294295936, offset 0 00000080:00000010:1.0:1775768274.610900:0:8:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99422030bf18. 00000400:00000200:2.0:1775768274.610901:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000080:00000010:1.0:1775768274.610902:0:8:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff994205751da8. 00000020:00000010:1.0:1775768274.610904:0:8:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994208a8f600. 00000400:00000200:2.0:1775768274.610905:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00000010:1.0:1775768274.610907:0:8:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff994210709ce0. 00000400:00000200:2.0:1775768274.610908:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00020000:00000010:1.0:1775768274.610909:0:8:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994205743180. 00000400:00000200:2.0:1775768274.610910:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00008000:00000010:1.0:1775768274.610911:0:8:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994219fec110. 00000400:00000200:2.0:1775768274.610913:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000080:00000010:1.0:1775768274.610914:0:8:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942057e2960. 00000020:00000010:1.0:1775768274.610916:0:8:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994208a8e000. 00000400:00000200:2.0:1775768274.610916:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000020:00000010:1.0:1775768274.610918:0:8:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff994206aba240. 00000400:00000200:2.0:1775768274.610918:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000080:00000001:1.0:1775768274.610920:0:8:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1775768274.610921:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225485:225485:256:4294967295] 192.168.202.156@tcp LPNI seq info [225481:225481:8:4294967295] 00000080:00000001:1.0:1775768274.610922:0:8:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768274.610925:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768274.610931:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768274.610935:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768274.610938:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768274.610942:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768274.610944:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775768274.610948:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768274.610950:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000100:2.0:1775768274.610954:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768274.610955:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768274.610958:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768274.610961:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768274.610963:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768274.610965:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768274.610967:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768274.610970:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a8700 x1862022294295936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768299 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768274.610978:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768274.610980:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a8700 x1862022294295936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768299 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768274.610984:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768274.610985:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768274.610987:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768274.610989:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:2.0:1775768274.610991:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768274.610992:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300199435 00000100:00000001:2.0:1775768274.610993:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300200765 00000100:00000001:2.0:1775768274.610995:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300199435 00000100:00000001:2.0:1775768274.610996:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300200765 00000100:00000001:2.0:1775768274.610998:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768274.610999:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768274.611000:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768274.611002:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768274/real 1775768274] req@ffff9942099a8700 x1862022294295936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768274.611006:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768274.611007:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768274.611008:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768274.611009:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768274.611010:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768274.611011:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768274.611013:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a8700 x1862022294295936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768274.611017:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a8700 x1862022294295936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768274.611021:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768274.611021:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768274.611022:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768274.611023:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768274.611025:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a8700 x1862022294295936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768274.611028:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768274.611030:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a8700 x1862022294295936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768299 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768274.611034:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768274.611035:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768274.611036:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768274.611036:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768274.611038:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768274.611040:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768274.611041:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768274.611043:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768274.611045:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768274.611047:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a8700 x1862022294295936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768299 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768274.611051:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294295936:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768274.611053:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768274.611054:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768274.611056:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a9180 x1862022294295808/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768299 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768274.611059:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768274.611060:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768274.611061:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f400. 02000000:00000001:2.0:1775768274.611062:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768274.611063:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768274.611063:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:2.0:1775768274.611064:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768274.611065:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b0000. 02000000:00000001:2.0:1775768274.611067:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768274.611067:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768274.611068:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a9180. 00000100:00000001:2.0:1775768274.611069:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768274.611070:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768274.611071:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768274.611073:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a8700 x1862022294295936/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768299 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768274.611076:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768274.611077:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768274.611078:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f000. 02000000:00000001:2.0:1775768274.611079:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768274.611080:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768274.611080:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:2.0:1775768274.611082:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768274.611082:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b2800. 02000000:00000001:2.0:1775768274.611084:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768274.611084:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768274.611085:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a8700. 00000100:00000001:2.0:1775768274.611086:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768274.611086:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768274.611087:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768274.611089:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768274.611090:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768274.611091:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768274.611092:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768274.611093:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768274.611094:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a8e00 x1862022294296064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768274.611098:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768274.611098:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768274.611100:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768274.611101:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768274.611102:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768274.611103:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294296064:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768274.611105:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768274.611106:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768274.611107:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768274.611108:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768274.611110:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f000. 02000000:00000001:2.0:1775768274.611111:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768274.611113:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768274.611114:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768274.611116:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768274.611118:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294296064, portal 4 00000100:00000001:2.0:1775768274.611119:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768274.611120:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640856064 : -112966068695552 : ffff9942099a8e00) 00000100:00000040:2.0:1775768274.611123:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a8e00 x1862022294296064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768299 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768274.611127:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768274.611128:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768274.611129:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768274.611131:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294296064, offset 0 00000400:00000200:2.0:1775768274.611133:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768274.611135:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768274.611136:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768274.611138:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768274.611140:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768274.611141:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768274.611143:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768274.611145:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225486:225486:256:4294967295] 192.168.202.156@tcp LPNI seq info [225482:225482:8:4294967295] 00000400:00000200:2.0:1775768274.611147:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768274.611151:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768274.611154:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768274.611169:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768274.611172:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768274.611174:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775768274.611176:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768274.611178:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000100:2.0:1775768274.611180:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768274.611181:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768274.611183:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768274.611185:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768274.611186:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768274.611187:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768274.611189:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768274.611191:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a8e00 x1862022294296064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768299 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768274.611196:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768274.611197:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a8e00 x1862022294296064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768299 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768274.611201:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768274.611202:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768274.611204:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768274.611206:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:2.0:1775768274.611207:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768274.611209:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300199459 00000100:00000001:2.0:1775768274.611210:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300200765 00000100:00000001:2.0:1775768274.611211:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300199459 00000100:00000001:2.0:1775768274.611213:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300200765 00000100:00000001:2.0:1775768274.611214:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768274.611215:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768274.611216:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768274.611218:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768274/real 1775768274] req@ffff9942099a8e00 x1862022294296064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768274.611222:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768274.611222:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768274.611223:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768274.611224:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768274.611225:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768274.611226:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768274.611227:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a8e00 x1862022294296064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768274.611232:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a8e00 x1862022294296064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768274.611235:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768274.611236:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768274.611251:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768274.611251:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768274.611254:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a8e00 x1862022294296064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768274.611258:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768274.611260:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a8e00 x1862022294296064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768299 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768274.611264:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768274.611265:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768274.611266:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768274.611267:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768274.611268:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768274.611270:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768274.611271:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768274.611273:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768274.611274:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768274.611276:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a8e00 x1862022294296064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768299 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768274.611280:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294296064:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768274.611282:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768274.611284:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a8380 x1862022294296192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768274.611287:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768274.611288:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768274.611289:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768274.611290:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768274.611291:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768274.611292:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294296192:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768274.611294:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768274.611295:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768274.611296:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768274.611297:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768274.611299:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f400. 02000000:00000001:2.0:1775768274.611300:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768274.611302:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768274.611303:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768274.611305:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768274.611307:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294296192, portal 4 00000100:00000001:2.0:1775768274.611308:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768274.611308:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640853376 : -112966068698240 : ffff9942099a8380) 00000100:00000040:2.0:1775768274.611312:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a8380 x1862022294296192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768299 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768274.611315:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768274.611316:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768274.611318:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768274.611319:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294296192, offset 0 00000400:00000200:2.0:1775768274.611321:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768274.611323:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768274.611324:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768274.611326:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768274.611327:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768274.611329:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768274.611330:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768274.611332:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225487:225487:256:4294967295] 192.168.202.156@tcp LPNI seq info [225483:225483:8:4294967295] 00000400:00000200:2.0:1775768274.611334:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768274.611338:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768274.611341:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768274.611342:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768274.611344:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768274.611346:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775768274.611348:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768274.611350:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 22 secs later) 00000800:00000100:2.0:1775768274.611352:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768274.611353:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768274.611355:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768274.611356:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768274.611358:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768274.611359:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768274.611360:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768274.611362:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a8380 x1862022294296192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768299 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768274.611366:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768274.611368:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a8380 x1862022294296192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768299 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768274.611371:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768274.611372:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768274.611373:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768274.611375:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:2.0:1775768274.611377:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768274.611378:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300199462 00000100:00000001:2.0:1775768274.611379:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300200765 00000100:00000001:2.0:1775768274.611380:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300199462 00000100:00000001:2.0:1775768274.611381:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300200765 00000100:00000001:2.0:1775768274.611382:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768274.611383:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768274.611384:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768274.611386:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768274/real 1775768274] req@ffff9942099a8380 x1862022294296192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768274.611390:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768274.611391:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768274.611392:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768274.611393:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768274.611394:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768274.611395:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768274.611397:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a8380 x1862022294296192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768274.611401:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a8380 x1862022294296192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768274.611405:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768274.611405:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768274.611406:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768274.611407:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768274.611408:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a8380 x1862022294296192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768274.611412:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768274.611414:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a8380 x1862022294296192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768299 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768274.611417:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768274.611418:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768274.611419:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768274.611420:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768274.611420:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768274.611422:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768274.611424:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768274.611425:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768274.611426:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768274.611428:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a8380 x1862022294296192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768299 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768274.611432:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294296192:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768274.611433:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768274.611435:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768274.611436:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a8e00 x1862022294296064/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768299 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768274.611440:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768274.611441:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768274.611442:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f000. 02000000:00000001:2.0:1775768274.611443:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768274.611443:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768274.611444:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:2.0:1775768274.611445:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768274.611446:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b2c00. 02000000:00000001:2.0:1775768274.611447:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768274.611448:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768274.611449:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a8e00. 00000100:00000001:2.0:1775768274.611450:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768274.611451:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768274.611452:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768274.611454:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a8380 x1862022294296192/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768299 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768274.611458:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768274.611458:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768274.611459:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f400. 02000000:00000001:2.0:1775768274.611460:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768274.611460:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768274.611461:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:2.0:1775768274.611462:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768274.611462:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b0c00. 02000000:00000001:2.0:1775768274.611464:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768274.611464:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768274.611465:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a8380. 00000100:00000001:2.0:1775768274.611466:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768274.611466:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768274.611467:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768274.611469:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768274.611469:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768274.611470:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768274.611471:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768274.673264:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768274.673268:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768274.673274:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768274.673275:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768274.673279:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768274.673280:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768274.673283:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768274.673285:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768274.673295:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768274.673296:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768274.673344:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768274.673347:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768274.673349:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768274.737268:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768274.737273:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768274.737274:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768274.737278:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768274.737280:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768274.737283:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768274.737285:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768274.737287:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768274.737289:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768275.633286:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768275.633286:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768275.633294:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768275.633294:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768275.633297:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768275.633298:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768275.633304:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768275.633304:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:2.0:1775768275.633304:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775768275.633310:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768275.633310:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768275.633311:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768275.633314:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768275.633316:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768275.633316:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768275.633317:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768275.633321:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768275.633322:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768275.633324:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768275.633327:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768275.633329:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768275.633329:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768275.633332:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768275.633335:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768275.633335:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768275.633339:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768275.633341:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768275.699807:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768275.699811:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768275.699816:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768275.699818:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768275.699821:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768275.699822:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768275.699824:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768275.699825:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768275.699828:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768275.699829:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768275.699845:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768275.699847:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768275.699849:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768275.761259:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768275.761264:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768275.761265:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768275.761269:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768275.761272:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768275.761275:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768275.761278:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768275.761280:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768275.761282:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768275.953272:0:15036:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4300200765, at 4300200768 00000100:00000040:1.0:1775768275.953272:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4300200765, at 4300200768 00000100:00000010:1.0:1775768275.953287:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d6de0. 00000100:00000010:0.0:1775768275.953289:0:15036:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d6ba0. 00000100:00000001:0.0:1775768276.657269:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768276.657270:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768276.657276:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768276.657277:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768276.657278:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768276.657280:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768276.657283:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768276.657284:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:2.0:1775768276.657285:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775768276.657286:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768276.657290:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768276.657290:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768276.657291:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768276.657293:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768276.657295:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768276.657295:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768276.657297:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768276.657299:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768276.657301:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768276.657302:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768276.657302:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768276.657304:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768276.657305:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768276.657309:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768276.657309:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768276.657312:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768276.657316:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768276.721237:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768276.721241:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768276.721245:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768276.721246:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768276.721249:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768276.721250:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768276.721252:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768276.721253:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768276.721255:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768276.721256:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768276.721300:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768276.721304:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768276.721308:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768276.785274:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768276.785278:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768276.785280:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768276.785283:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768276.785286:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768276.785289:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768276.785290:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768276.785292:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768276.785294:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768277.681267:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768277.681272:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768277.681274:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768277.681279:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768277.681279:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768277.681283:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768277.681285:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768277.681286:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768277.681288:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768277.681289:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768277.681290:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768277.681292:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775768277.681293:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768277.681294:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768277.681297:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768277.681297:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768277.681298:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:0.0:1775768277.681303:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768277.681303:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768277.681308:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768277.681308:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768277.681310:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768277.681311:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768277.681316:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768277.681316:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768277.681319:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768277.681321:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768277.745267:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768277.745272:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768277.745276:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768277.745279:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768277.745280:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768277.745283:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768277.745285:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768277.745287:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768277.745288:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768277.745289:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768277.745292:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768277.745293:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768277.745294:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768277.745295:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768277.745295:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768277.745296:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768277.745297:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768277.745298:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768277.745300:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768277.745301:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768277.745302:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768277.745303:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768277.745303:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768277.745304:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768277.745305:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768277.745306:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768277.745308:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768277.745309:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768277.745310:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768277.745310:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768277.745311:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768277.745312:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768277.745313:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768277.745314:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768277.745326:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768277.745329:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768277.745331:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768277.809232:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768277.809236:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768277.809237:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768277.809241:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768277.809248:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768277.809251:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768277.809253:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768277.809255:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768277.809256:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768278.705265:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768278.705274:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768278.705276:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768278.705282:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768278.705287:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768278.705291:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768278.705294:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768278.705296:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768278.705299:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768278.705386:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768278.705389:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768278.705390:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768278.705393:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768278.705395:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768278.705397:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768278.705398:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768278.705400:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768278.705401:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768278.705972:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768278.705977:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768278.705979:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768278.705982:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768278.705984:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768278.705986:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768278.705988:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768278.705989:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768278.705990:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768278.769298:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768278.769307:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768278.769315:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768278.769317:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768278.769321:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768278.769322:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768278.769326:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768278.769327:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768278.769331:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768278.769332:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768278.769392:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768278.769395:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768278.769398:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768278.833303:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768278.833312:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768278.833314:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768278.833321:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768278.833325:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768278.833330:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768278.833333:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768278.833336:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768278.833339:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768279.729274:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768279.729274:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768279.729274:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768279.729282:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768279.729282:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768279.729282:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768279.729285:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768279.729286:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768279.729292:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768279.729293:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775768279.729293:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:2.0:1775768279.729297:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768279.729298:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768279.729299:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768279.729301:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768279.729304:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768279.729304:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768279.729305:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768279.729308:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768279.729311:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768279.729314:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768279.729315:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768279.729317:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768279.729320:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768279.729322:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768279.729325:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768279.729328:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768279.729356:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768279.729361:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775768279.729364:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775768279.729367:0:36:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775768279.729371:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775768279.729378:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5532 00000400:00000200:3.0:1775768279.729387:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775768279.729393:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775768279.729396:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775768279.729399:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768279.729403:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775768279.729405:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768279.729406:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775768279.729409:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775768279.729410:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:3.0:1775768279.729413:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775768279.729415:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:3.0:1775768279.729416:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:3.0:1775768279.729418:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:3.0:1775768279.729420:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768279.729421:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775768279.729422:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768279.729423:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775768279.729426:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:3.0:1775768279.729429:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775768279.729432:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768279.729434:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775768279.729437:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775768279.729439:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768279.729440:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775768279.729442:0:36:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775768279.729452:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8380. 00000020:00000040:3.0:1775768279.729456:0:36:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:3.0:1775768279.729461:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775768279.729463:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768279.729466:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775768279.729469:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775768279.729475:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775768279.729486:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b1000. 00000100:00000001:3.0:1775768279.729493:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768279.729503:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8380 x1862022294296320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775768279.729520:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775768279.729526:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768279.729531:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000055f5621a] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8380 x1862022294296320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768279.729552:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768279.729555:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768279.729558:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775768279.729560:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775768279.729562:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775768279.729564:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775768279.729567:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5532 00000400:00000200:3.0:1775768279.729572:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775768279.729576:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:2.0:1775768279.729578:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1775768279.729579:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1775768279.729582:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1775768279.729584:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768279.729585:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1775768279.729588:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775768279.729592:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000040:2.0:1775768279.729593:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a8380 x1862022294296320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768279.729596:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775768279.729600:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775768279.729602:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:3.0:1775768279.729605:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775768279.729608:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000100:00000001:2.0:1775768279.729611:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000001:3.0:1775768279.729612:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000001:2.0:1775768279.729615:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768279.729616:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 02000000:00000001:2.0:1775768279.729620:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:3.0:1775768279.729621:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768279.729624:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1775768279.729625:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1775768279.729627:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000001:2.0:1775768279.729630:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768279.729631:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775768279.729635:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00080000:3.0:1775768279.729638:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775768279.729642:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768279.729643:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294296320:192.168.202.156@tcp:250:kworker.0 02000000:00000001:3.0:1775768279.729644:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1775768279.729650:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775768279.729650:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775768279.729653:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768279.729654:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:2.0:1775768279.729655:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768279.729659:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1775768279.729659:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1775768279.729663:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:3.0:1775768279.729663:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768279.729665:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775768279.729668:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8e00. 02000000:00000010:2.0:1775768279.729670:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f400. 00000020:00000040:3.0:1775768279.729670:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:3.0:1775768279.729674:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775768279.729675:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768279.729676:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768279.729679:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775768279.729681:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775768279.729682:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768279.729685:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000010:3.0:1775768279.729686:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b2000. 00000100:00000001:3.0:1775768279.729690:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775768279.729691:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000100:00100000:3.0:1775768279.729694:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8e00 x1862022294296448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775768279.729700:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000010:2.0:1775768279.729701:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000001:3.0:1775768279.729702:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768279.729706:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000864a6866] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8e00 x1862022294296448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768279.729712:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294296320, portal 25 00000100:00000001:3.0:1775768279.729712:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768279.729716:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775768279.729717:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1775768279.729719:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775768279.729720:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640853376 : -112966068698240 : ffff9942099a8380) 00000100:00000001:3.0:1775768279.729721:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775768279.729723:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775768279.729726:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775768279.729728:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5532 00000100:00000040:2.0:1775768279.729732:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a8380 x1862022294296320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768304 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1775768279.729732:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775768279.729736:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775768279.729739:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775768279.729742:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768279.729746:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775768279.729749:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768279.729751:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1775768279.729751:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775768279.729753:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775768279.729755:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:3.0:1775768279.729757:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000040:2.0:1775768279.729758:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000020:00000040:3.0:1775768279.729760:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000400:00000010:2.0:1775768279.729762:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000020:00000001:3.0:1775768279.729762:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:3.0:1775768279.729764:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000200:2.0:1775768279.729767:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294296320, offset 0 00000100:00000040:3.0:1775768279.729768:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768279.729770:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775768279.729771:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768279.729773:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000200:2.0:1775768279.729776:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000020:00000040:3.0:1775768279.729776:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:3.0:1775768279.729780:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775768279.729781:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768279.729782:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775768279.729784:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775768279.729785:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768279.729788:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775768279.729789:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775768279.729791:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775768279.729793:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768279.729794:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768279.729796:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00000010:3.0:1775768279.729798:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a8700. 00000020:00000040:3.0:1775768279.729800:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:3.0:1775768279.729801:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000200:2.0:1775768279.729803:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:3.0:1775768279.729803:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768279.729806:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775768279.729807:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:2.0:1775768279.729809:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 02000000:00000001:3.0:1775768279.729811:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768279.729814:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 02000000:00000010:3.0:1775768279.729815:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b3400. 00000400:00000200:2.0:1775768279.729817:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00000001:3.0:1775768279.729821:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768279.729823:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225488:225488:256:4294967295] 192.168.202.156@tcp LPNI seq info [225484:225484:8:4294967295] 00000400:00000200:2.0:1775768279.729828:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00100000:3.0:1775768279.729830:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a8700 x1862022294296576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1775768279.729841:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000100:00000001:3.0:1775768279.729846:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775768279.729849:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1775768279.729850:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768279.729856:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000100:00000040:3.0:1775768279.729857:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000faa25e3b] to pc [ptlrpcd_rcv+-1] req@ffff9942099a8700 x1862022294296576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1775768279.729867:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:3.0:1775768279.729867:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775768279.729871:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:2.0:1775768279.729872:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000100:00080000:3.0:1775768279.729875:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775768279.729877:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775768279.729879:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:2.1:1775768279.729879:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:3.0:1775768279.729882:0:36:0:(import.c:484:import_select_connection()) Process entered 00000800:00000200:2.1:1775768279.729884:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000100:00080000:3.0:1775768279.729887:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5532 00000800:00000100:2.0:1775768279.729893:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000400:00000200:3.0:1775768279.729893:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775768279.729897:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000800:00000010:2.0:1775768279.729898:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000100:00080000:3.0:1775768279.729899:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775768279.729903:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768279.729904:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775768279.729906:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000200:2.0:1775768279.729908:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:3.0:1775768279.729908:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775768279.729909:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775768279.729911:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:3.0:1775768279.729913:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775768279.729915:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:3.0:1775768279.729917:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000400:00000200:2.0:1775768279.729920:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000020:00000001:3.0:1775768279.729920:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:3.0:1775768279.729925:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768279.729927:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775768279.729928:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000200:2.0:1775768279.729930:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000100:00000001:3.0:1775768279.729930:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:3.0:1775768279.729934:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000400:00000010:2.0:1775768279.729935:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00080000:3.0:1775768279.729936:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775768279.729939:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768279.729940:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1775768279.729942:0:2399:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:3.0:1775768279.729942:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775768279.729943:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768279.729946:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775768279.729948:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000100:00000200:2.0:1775768279.729950:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a8380 x1862022294296320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768304 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1775768279.729951:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768279.729953:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775768279.729956:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099a9180. 00000020:00000040:3.0:1775768279.729959:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:3.0:1775768279.729964:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775768279.729965:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768279.729968:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775768279.729970:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775768279.729971:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768279.729972:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 02000000:00000010:3.0:1775768279.729974:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0400. 00000100:00000040:2.0:1775768279.729978:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a8380 x1862022294296320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768304 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768279.729978:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768279.729984:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099a9180 x1862022294296704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775768279.729997:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768279.730000:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775768279.730000:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775768279.730002:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768279.730005:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bdc2586] to pc [ptlrpcd_rcv+-1] req@ffff9942099a9180 x1862022294296704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768279.730007:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768279.730013:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768279.730016:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00080000:3.0:1775768279.730017:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775768279.730019:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4300205885 00000100:00080000:3.0:1775768279.730019:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775768279.730022:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775768279.730023:0:36:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775768279.730024:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00000010:2.0:1775768279.730026:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d6f60. 00000100:00080000:3.0:1775768279.730026:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5532 00000400:00000200:3.0:1775768279.730031:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:3.0:1775768279.730032:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:3.0:1775768279.730034:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1775768279.730035:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000040:3.0:1775768279.730036:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768279.730039:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775768279.730041:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768279.730042:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4300205885 00000100:00000001:3.0:1775768279.730043:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775768279.730046:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000010:2.0:1775768279.730047:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d67e0. 00000020:00000040:3.0:1775768279.730047:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:3.0:1775768279.730050:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:2.0:1775768279.730051:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000020:00000040:3.0:1775768279.730051:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:3.0:1775768279.730055:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:3.0:1775768279.730056:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000001:2.0:1775768279.730057:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1775768279.730059:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768279.730062:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768279.730062:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775768279.730063:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:3.0:1775768279.730066:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:2.0:1775768279.730068:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000020:00000040:3.0:1775768279.730068:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:3.0:1775768279.730072:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:3.0:1775768279.730073:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775768279.730074:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775768279.730076:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775768279.730077:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768279.730078:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775768279.730079:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:3.0:1775768279.730080:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768279.730081:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775768279.730083:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ab480. 00000100:00100000:2.0:1775768279.730085:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768279/real 1775768279] req@ffff9942099a8380 x1862022294296320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1775768279.730085:0:36:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:3.0:1775768279.730088:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775768279.730089:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768279.730092:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775768279.730094:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775768279.730095:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775768279.730099:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9942054b0c00. 00000100:00000001:3.0:1775768279.730102:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1775768279.730106:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00100000:3.0:1775768279.730106:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ab480 x1862022294296832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:2.0:1775768279.730110:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768279.730114:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768279.730117:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768279.730119:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768279.730122:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1775768279.730128:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000200:2.0:1775768279.730129:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a8380 x1862022294296320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768279.730130:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768279.730137:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000067a0ba74] to pc [ptlrpcd_rcv+-1] req@ffff9942099ab480 x1862022294296832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768279.730146:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a8380 x1862022294296320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775768279.730146:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775768279.730150:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5542) 00000100:00000001:2.0:1775768279.730170:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768279.730171:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768279.730173:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768279.730175:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768279.730178:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a8380 x1862022294296320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768279.730183:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768279.730186:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a8380 x1862022294296320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768304 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768279.730206:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768279.730208:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768279.730209:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768279.730210:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768279.730212:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768279.730214:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768279.730215:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768279.730218:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768279.730220:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768279.730222:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a8380 x1862022294296320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768304 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768279.730227:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294296320:192.168.202.156@tcp:250:kworker.0 00000100:00000001:2.0:1775768279.730230:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768279.730232:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768279.730233:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a8380 x1862022294296320/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768304 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768279.730237:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768279.730239:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768279.730242:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f400. 02000000:00000001:2.0:1775768279.730244:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768279.730245:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768279.730246:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:2.0:1775768279.730247:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768279.730249:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b1000. 02000000:00000001:2.0:1775768279.730253:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768279.730254:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768279.730255:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a8380. 00000100:00000001:2.0:1775768279.730257:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768279.730258:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768279.730259:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768279.730266:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768279.730267:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768279.730268:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768279.730269:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768279.730270:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768279.730272:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a8e00 x1862022294296448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768279.730277:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768279.730278:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768279.730279:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768279.730280:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768279.730281:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768279.730283:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294296448:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768279.730285:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768279.730286:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768279.730287:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768279.730288:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768279.730290:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f400. 02000000:00000001:2.0:1775768279.730291:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768279.730294:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768279.730296:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768279.730299:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768279.730301:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294296448, portal 10 00000100:00000001:2.0:1775768279.730302:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768279.730303:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640856064 : -112966068695552 : ffff9942099a8e00) 00000100:00000040:2.0:1775768279.730308:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a8e00 x1862022294296448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768304 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768279.730312:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768279.730314:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768279.730315:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768279.730316:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294296448, offset 0 00000400:00000200:2.0:1775768279.730319:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768279.730321:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768279.730324:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768279.730326:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768279.730328:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768279.730329:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768279.730331:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768279.730333:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225489:225489:256:4294967295] 192.168.202.156@tcp LPNI seq info [225485:225485:8:4294967295] 00000400:00000200:2.0:1775768279.730336:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768279.730339:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768279.730342:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768279.730344:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768279.730346:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768279.730348:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775768279.730351:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768279.730353:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000100:2.0:1775768279.730355:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768279.730356:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768279.730358:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768279.730360:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768279.730362:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768279.730363:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768279.730365:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768279.730367:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a8e00 x1862022294296448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768304 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768279.730372:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768279.730373:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a8e00 x1862022294296448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768304 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768279.730377:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768279.730378:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768279.730380:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768279.730383:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:2.0:1775768279.730385:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768279.730386:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300204563 00000100:00000001:2.0:1775768279.730389:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300205885 00000100:00000001:2.0:1775768279.730390:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300204563 00000100:00000001:2.0:1775768279.730391:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300205885 00000100:00000001:2.0:1775768279.730393:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768279.730394:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768279.730395:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768279.730397:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768279/real 1775768279] req@ffff9942099a8e00 x1862022294296448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768279.730401:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768279.730401:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768279.730402:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768279.730404:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768279.730405:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768279.730406:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768279.730407:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a8e00 x1862022294296448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768279.730412:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a8e00 x1862022294296448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768279.730415:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768279.730416:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768279.730417:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768279.730417:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768279.730419:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a8e00 x1862022294296448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768279.730422:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768279.730424:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a8e00 x1862022294296448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768304 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768279.730429:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768279.730430:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768279.730431:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768279.730432:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768279.730433:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768279.730435:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768279.730437:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768279.730439:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768279.730440:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768279.730442:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a8e00 x1862022294296448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768304 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768279.730447:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294296448:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768279.730449:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768279.730451:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a8700 x1862022294296576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768279.730454:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768279.730455:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768279.730456:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768279.730457:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768279.730458:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768279.730459:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294296576:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768279.730460:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768279.730461:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768279.730462:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768279.730463:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768279.730465:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f000. 02000000:00000001:2.0:1775768279.730466:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768279.730468:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768279.730469:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768279.730471:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768279.730472:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294296576, portal 10 00000100:00000001:2.0:1775768279.730474:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768279.730475:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640854272 : -112966068697344 : ffff9942099a8700) 00000100:00000040:2.0:1775768279.730478:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a8700 x1862022294296576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768304 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768279.730482:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768279.730484:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768279.730485:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768279.730486:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294296576, offset 0 00000400:00000200:2.0:1775768279.730488:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768279.730490:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768279.730492:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768279.730494:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768279.730495:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768279.730497:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768279.730505:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768279.730509:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225490:225490:256:4294967295] 192.168.202.156@tcp LPNI seq info [225486:225486:8:4294967295] 00000400:00000200:2.0:1775768279.730517:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768279.730523:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768279.730527:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768279.730531:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768279.730535:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768279.730537:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775768279.730541:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768279.730543:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000100:2.0:1775768279.730547:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768279.730549:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768279.730552:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768279.730555:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768279.730557:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768279.730558:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768279.730560:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768279.730562:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a8700 x1862022294296576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768304 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768279.730570:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768279.730572:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a8700 x1862022294296576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768304 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768279.730576:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768279.730577:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768279.730578:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768279.730580:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:2.0:1775768279.730582:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768279.730583:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300204555 00000100:00000001:2.0:1775768279.730585:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300205885 00000100:00000001:2.0:1775768279.730586:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300204555 00000100:00000001:2.0:1775768279.730587:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300205885 00000100:00000001:2.0:1775768279.730589:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768279.730590:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768279.730591:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768279.730593:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768279/real 1775768279] req@ffff9942099a8700 x1862022294296576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768279.730597:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768279.730597:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768279.730599:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768279.730600:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768279.730602:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768279.730604:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768279.730606:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a8700 x1862022294296576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768279.730614:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a8700 x1862022294296576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768279.730620:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768279.730622:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768279.730623:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768279.730625:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768279.730628:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a8700 x1862022294296576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768279.730634:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768279.730638:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a8700 x1862022294296576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768304 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768279.730645:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768279.730647:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768279.730649:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768279.730650:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768279.730652:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768279.730654:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768279.730657:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768279.730659:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768279.730661:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768279.730663:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a8700 x1862022294296576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768304 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768279.730668:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294296576:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768279.730671:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768279.730674:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099a9180 x1862022294296704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768279.730680:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768279.730681:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768279.730683:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768279.730685:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768279.730687:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768279.730690:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099a9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294296704:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768279.730693:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768279.730695:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768279.730696:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768279.730698:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768279.730701:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1ec00. 02000000:00000001:2.0:1775768279.730703:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768279.730707:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768279.730709:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768279.730712:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768279.730715:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294296704, portal 4 00000100:00000001:2.0:1775768279.730717:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768279.730718:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640856960 : -112966068694656 : ffff9942099a9180) 00000100:00000040:2.0:1775768279.730724:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099a9180 x1862022294296704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768304 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768279.730730:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768279.730732:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768279.730735:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768279.730737:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294296704, offset 0 00000400:00000200:2.0:1775768279.730740:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768279.730744:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768279.730747:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768279.730749:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768279.730751:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768279.730752:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768279.730754:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768279.730755:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225491:225491:256:4294967295] 192.168.202.156@tcp LPNI seq info [225487:225487:8:4294967295] 00000400:00000200:2.0:1775768279.730758:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768279.730762:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768279.730765:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768279.730767:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768279.730770:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768279.730771:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775768279.730774:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768279.730776:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000100:2.0:1775768279.730778:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768279.730779:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768279.730781:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768279.730783:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768279.730785:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768279.730786:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768279.730787:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768279.730790:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099a9180 x1862022294296704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768304 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768279.730795:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768279.730796:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099a9180 x1862022294296704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768304 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768279.730800:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768279.730801:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768279.730803:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768279.730805:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:2.0:1775768279.730806:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768279.730808:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300204578 00000100:00000001:2.0:1775768279.730809:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300205885 00000100:00000001:2.0:1775768279.730811:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300204578 00000100:00000001:2.0:1775768279.730812:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300205885 00000100:00000001:2.0:1775768279.730813:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768279.730814:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768279.730815:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768279.730818:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768279/real 1775768279] req@ffff9942099a9180 x1862022294296704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768279.730825:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768279.730826:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768279.730828:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768279.730830:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768279.730831:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768279.730833:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768279.730836:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099a9180 x1862022294296704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768279.730844:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099a9180 x1862022294296704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768279.730850:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768279.730851:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768279.730853:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768279.730854:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768279.730858:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099a9180 x1862022294296704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768279.730864:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768279.730868:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099a9180 x1862022294296704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768304 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768279.730875:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768279.730877:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768279.730878:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768279.730880:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768279.730881:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768279.730885:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768279.730888:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768279.730891:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768279.730893:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768279.730897:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099a9180 x1862022294296704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768304 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768279.730911:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099a9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294296704:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768279.730915:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768279.730916:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ab480 x1862022294296832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768279.730921:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768279.730921:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768279.730923:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768279.730924:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768279.730925:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768279.730927:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ab480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294296832:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768279.730929:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768279.730930:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768279.730931:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768279.730932:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768279.730934:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1fc00. 02000000:00000001:2.0:1775768279.730935:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768279.730938:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768279.730939:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768279.730941:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768279.730944:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294296832, portal 4 00000100:00000001:2.0:1775768279.730945:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768279.730947:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107640865920 : -112966068685696 : ffff9942099ab480) 00000100:00000040:2.0:1775768279.730951:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ab480 x1862022294296832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768304 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768279.730958:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768279.730960:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768279.730962:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768279.730964:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294296832, offset 0 00000400:00000200:2.0:1775768279.730967:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768279.730972:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768279.730975:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768279.730978:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768279.730980:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768279.730982:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768279.730984:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768279.730987:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225492:225492:256:4294967295] 192.168.202.156@tcp LPNI seq info [225488:225488:8:4294967295] 00000400:00000200:2.0:1775768279.730992:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768279.730997:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768279.731002:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768279.731005:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768279.731008:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768279.731011:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775768279.731015:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768279.731017:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 17 secs later) 00000800:00000100:2.0:1775768279.731021:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768279.731022:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768279.731024:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768279.731026:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768279.731028:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768279.731029:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768279.731031:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768279.731033:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ab480 x1862022294296832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768304 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768279.731038:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768279.731040:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ab480 x1862022294296832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768304 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768279.731044:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768279.731045:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768279.731046:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768279.731048:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:2.0:1775768279.731050:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768279.731052:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300204581 00000100:00000001:2.0:1775768279.731054:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300205885 00000100:00000001:2.0:1775768279.731055:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300204581 00000100:00000001:2.0:1775768279.731057:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300205885 00000100:00000001:2.0:1775768279.731058:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768279.731059:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768279.731060:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768279.731062:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768279/real 1775768279] req@ffff9942099ab480 x1862022294296832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768279.731066:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768279.731067:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768279.731068:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768279.731070:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768279.731070:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768279.731071:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768279.731073:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ab480 x1862022294296832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768279.731078:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ab480 x1862022294296832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768279.731081:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768279.731082:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768279.731083:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768279.731084:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768279.731085:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ab480 x1862022294296832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768279.731089:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768279.731091:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ab480 x1862022294296832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768304 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768279.731094:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768279.731096:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768279.731097:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768279.731098:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768279.731099:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768279.731102:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768279.731103:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768279.731105:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768279.731106:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768279.731108:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ab480 x1862022294296832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768304 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768279.731112:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ab480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294296832:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768279.731114:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768279.731115:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768279.731117:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a8e00 x1862022294296448/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768304 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768279.731121:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768279.731122:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768279.731123:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f400. 02000000:00000001:2.0:1775768279.731124:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768279.731125:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768279.731125:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:2.0:1775768279.731126:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768279.731127:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b2000. 02000000:00000001:2.0:1775768279.731129:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768279.731130:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768279.731131:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a8e00. 00000100:00000001:2.0:1775768279.731133:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768279.731133:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768279.731134:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768279.731136:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a8700 x1862022294296576/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768304 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768279.731144:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768279.731144:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768279.731145:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f000. 02000000:00000001:2.0:1775768279.731147:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768279.731147:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768279.731148:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:2.0:1775768279.731149:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768279.731149:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b3400. 02000000:00000001:2.0:1775768279.731151:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768279.731151:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768279.731152:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a8700. 00000100:00000001:2.0:1775768279.731153:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768279.731154:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768279.731168:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768279.731170:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099a9180 x1862022294296704/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768304 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768279.731174:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768279.731174:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768279.731175:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1ec00. 02000000:00000001:2.0:1775768279.731178:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768279.731178:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768279.731179:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:2.0:1775768279.731180:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768279.731180:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b0400. 02000000:00000001:2.0:1775768279.731182:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768279.731183:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768279.731184:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099a9180. 00000100:00000001:2.0:1775768279.731185:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768279.731186:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768279.731187:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768279.731199:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ab480 x1862022294296832/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768304 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768279.731203:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768279.731204:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768279.731205:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1fc00. 02000000:00000001:2.0:1775768279.731206:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768279.731207:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768279.731208:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:2.0:1775768279.731209:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768279.731210:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9942054b0c00. 02000000:00000001:2.0:1775768279.731211:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768279.731212:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768279.731212:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ab480. 00000100:00000001:2.0:1775768279.731214:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768279.731215:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768279.731216:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768279.731218:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768279.731219:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768279.731220:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768279.731221:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768279.793307:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768279.793314:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768279.793321:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768279.793323:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768279.793327:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768279.793328:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768279.793332:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768279.793333:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768279.793338:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768279.793339:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768279.793360:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768279.793364:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768279.793367:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768279.857242:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768279.857250:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768279.857251:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768279.857256:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768279.857259:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768279.857263:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768279.857265:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768279.857267:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768279.857270:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768280.753337:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768280.753337:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768280.753345:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768280.753346:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768280.753349:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768280.753349:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768280.753357:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768280.753362:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768280.753368:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768280.753371:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768280.753373:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775768280.753377:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768280.753381:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768280.753384:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768280.753447:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768280.753447:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768280.753450:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768280.753454:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768280.753477:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768280.753482:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768280.753483:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768280.753486:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775768280.753489:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768280.753491:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768280.753493:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768280.753495:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768280.753497:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768280.817306:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768280.817313:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768280.817321:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768280.817323:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768280.817328:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768280.817330:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768280.817333:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768280.817334:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768280.817338:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768280.817339:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768280.817398:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768280.817401:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768280.817404:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768280.881301:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768280.881310:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768280.881312:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768280.881319:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768280.881323:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768280.881327:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768280.881330:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768280.881333:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768280.881336:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768281.073245:0:15036:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4300205885, at 4300205888 00000100:00000040:1.0:1775768281.073245:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4300205885, at 4300205888 00000100:00000010:0.0:1775768281.073261:0:15036:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d6f60. 00000100:00000010:1.0:1775768281.073265:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d67e0. 00000100:00000001:3.0:1775768281.777242:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768281.777250:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768281.777252:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768281.777258:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768281.777264:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:3.0:1775768281.777264:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768281.777267:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768281.777269:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1775768281.777271:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775768281.777273:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768281.777274:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768281.777276:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768281.777278:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768281.777279:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768281.777284:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768281.777284:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768281.777288:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768281.777292:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768281.777503:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768281.777507:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768281.777508:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768281.777511:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775768281.777513:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768281.777515:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768281.777517:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768281.777518:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768281.777520:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768281.841333:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768281.841341:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768281.841343:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768281.841346:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768281.841350:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768281.841353:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768281.841354:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768281.841356:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768281.841358:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768281.841359:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768281.841362:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768281.841363:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768281.841365:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768281.841366:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768281.841369:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768281.841369:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768281.841410:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768281.841414:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768281.841417:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768281.905332:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768281.905341:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768281.905343:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768281.905351:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768281.905355:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768281.905359:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768281.905362:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768281.905365:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768281.905369:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768282.801227:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768282.801235:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768282.801237:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768282.801243:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768282.801247:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768282.801249:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768282.801253:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768282.801254:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768282.801257:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768282.801258:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768282.801261:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768282.801261:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768282.801264:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768282.801265:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768282.801268:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768282.801269:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768282.801271:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768282.801274:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768282.801279:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768282.801285:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768282.801287:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768282.801292:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775768282.801296:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768282.801298:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768282.801300:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768282.801302:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768282.801304:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768282.865288:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768282.865296:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768282.865303:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768282.865304:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768282.865309:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768282.865311:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768282.865315:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768282.865316:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768282.865320:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768282.865322:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768282.865392:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768282.865396:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768282.865399:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768282.929298:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768282.929306:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768282.929308:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768282.929327:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768282.929331:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768282.929336:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768282.929340:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768282.929343:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768282.929346:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768283.825246:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768283.825251:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768283.825254:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768283.825256:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768283.825259:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768283.825259:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768283.825268:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775768283.825268:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768283.825274:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768283.825275:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768283.825280:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768283.825281:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768283.825285:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768283.825287:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768283.825291:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768283.825291:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768283.825294:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768283.825298:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768283.825304:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768283.825312:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768283.825314:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768283.825320:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775768283.825324:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768283.825328:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768283.825330:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768283.825332:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768283.825335:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768283.889267:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768283.889271:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768283.889275:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768283.889277:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768283.889281:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768283.889283:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768283.889286:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768283.889287:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768283.889290:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768283.889292:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768283.889321:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768283.889325:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768283.889328:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768283.953297:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768283.953304:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768283.953305:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768283.953311:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768283.953314:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768283.953318:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768283.953320:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768283.953322:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768283.953324:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768284.849231:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768284.849236:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768284.849239:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768284.849246:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768284.849249:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768284.849251:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768284.849253:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768284.849256:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768284.849256:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768284.849261:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775768284.849262:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768284.849265:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:2.0:1775768284.849266:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1775768284.849266:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768284.849267:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768284.849276:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768284.849276:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768284.849276:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768284.849281:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768284.849283:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:2.0:1775768284.849284:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:0.0:1775768284.849288:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768284.849289:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768284.849290:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1775768284.849295:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768284.849297:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768284.849298:0:36:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768284.849299:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768284.849306:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768284.849309:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768284.849310:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:0.0:1775768284.849316:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768284.849321:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5537 00000400:00000200:2.0:1775768284.849331:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768284.849340:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775768284.849343:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775768284.849346:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768284.849350:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768284.849353:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768284.849355:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775768284.849359:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775768284.849360:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:2.0:1775768284.849363:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775768284.849366:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:2.0:1775768284.849368:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:2.0:1775768284.849370:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:2.0:1775768284.849373:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768284.849374:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768284.849381:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768284.849382:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775768284.849384:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:2.0:1775768284.849387:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775768284.849391:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768284.849393:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775768284.849395:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768284.849397:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768284.849399:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775768284.849401:0:36:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775768284.849406:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfdc00. 00000020:00000040:2.0:1775768284.849408:0:36:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:2.0:1775768284.849411:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775768284.849412:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768284.849414:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775768284.849417:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775768284.849420:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775768284.849426:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1fc00. 00000100:00000001:2.0:1775768284.849431:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768284.849438:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfdc00 x1862022294296960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775768284.849445:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775768284.849448:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768284.849451:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003e25603c] to pc [ptlrpcd_rcv+-1] req@ffff994207dfdc00 x1862022294296960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768284.849469:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768284.849471:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768284.849474:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775768284.849475:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775768284.849476:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775768284.849478:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775768284.849479:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5537 00000400:00000200:2.0:1775768284.849482:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768284.849484:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775768284.849486:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775768284.849487:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768284.849489:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768284.849490:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768284.849492:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775768284.849493:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775768284.849494:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:2.0:1775768284.849495:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775768284.849497:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:2.0:1775768284.849498:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:2.0:1775768284.849499:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:2.0:1775768284.849501:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768284.849502:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768284.849503:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768284.849505:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775768284.849506:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00080000:2.0:1775768284.849508:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775768284.849510:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768284.849511:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1775768284.849511:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775768284.849515:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768284.849516:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1775768284.849517:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768284.849520:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775768284.849521:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000008:00100000:2.0:1775768284.849523:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775768284.849525:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768284.849528:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768284.849530:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfdc00 x1862022294296960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1775768284.849531:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dff100. 00000020:00000040:2.0:1775768284.849534:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:2.0:1775768284.849538:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775768284.849539:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768284.849543:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775768284.849545:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775768284.849547:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775768284.849551:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1ec00. 00000100:00000001:0.0:1775768284.849552:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768284.849555:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768284.849555:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768284.849561:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00100000:2.0:1775768284.849564:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dff100 x1862022294297088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1775768284.849565:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768284.849570:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768284.849578:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294296960:192.168.202.156@tcp:250:kworker.0 00000100:00000001:2.0:1775768284.849579:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775768284.849583:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768284.849587:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000040:2.0:1775768284.849590:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3e64ae1] to pc [ptlrpcd_rcv+-1] req@ffff994207dff100 x1862022294297088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768284.849600:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768284.849601:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:2.0:1775768284.849603:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:0.0:1775768284.849604:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768284.849610:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:2.0:1775768284.849610:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775768284.849613:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775768284.849615:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000010:0.0:1775768284.849618:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002a800. 00000100:00000001:2.0:1775768284.849618:0:36:0:(import.c:484:import_select_connection()) Process entered 02000000:00000001:0.0:1775768284.849623:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768284.849623:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5537 00000400:00000200:2.0:1775768284.849630:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768284.849633:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768284.849635:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775768284.849638:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000400:00000010:0.0:1775768284.849639:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbc268. 00000100:00000040:2.0:1775768284.849640:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768284.849644:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768284.849646:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768284.849648:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000010:0.0:1775768284.849649:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014ee0. 00000020:00000001:2.0:1775768284.849649:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775768284.849653:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:2.0:1775768284.849655:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775768284.849657:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000100:00000200:0.0:1775768284.849658:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294296960, portal 25 00000020:00000001:2.0:1775768284.849659:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000001:0.0:1775768284.849663:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000020:00000001:2.0:1775768284.849663:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000001:0.0:1775768284.849666:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611843584 : -112966097708032 : ffff994207dfdc00) 00000100:00000040:2.0:1775768284.849669:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768284.849672:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768284.849676:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfdc00 x1862022294296960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768309 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775768284.849676:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768284.849678:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775768284.849683:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:2.0:1775768284.849689:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775768284.849695:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768284.849696:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:2.0:1775768284.849698:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:0.0:1775768284.849702:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 02000000:00000001:2.0:1775768284.849702:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:0.0:1775768284.849705:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 02000000:00000001:2.0:1775768284.849706:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768284.849710:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000200:0.0:1775768284.849712:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294296960, offset 0 00000008:00100000:2.0:1775768284.849712:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775768284.849717:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768284.849718:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768284.849721:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00000010:2.0:1775768284.849722:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfc700. 00000020:00000040:2.0:1775768284.849724:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:2.0:1775768284.849726:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000200:0.0:1775768284.849728:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:2.0:1775768284.849728:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768284.849732:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775768284.849733:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775768284.849735:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768284.849736:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768284.849741:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768284.849744:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768284.849746:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768284.849749:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768284.849751:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225493:225493:256:4294967295] 192.168.202.156@tcp LPNI seq info [225489:225489:8:4294967295] 02000000:00000010:2.0:1775768284.849755:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1f000. 00000400:00000200:0.0:1775768284.849756:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768284.849762:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768284.849770:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000100:00100000:2.0:1775768284.849770:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfc700 x1862022294297216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:0.0:1775768284.849783:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00000001:2.0:1775768284.849783:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775768284.849787:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1775768284.849790:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6500. 00000100:00000040:2.0:1775768284.849793:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a176dd1] to pc [ptlrpcd_rcv+-1] req@ffff994207dfc700 x1862022294297216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1775768284.849800:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768284.849804:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775768284.849820:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:2.0:1775768284.849821:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768284.849826:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:0.1:1775768284.849826:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000100:00080000:2.0:1775768284.849834:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1775768284.849837:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000800:00000100:0.0:1775768284.849838:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000100:00080000:2.0:1775768284.849841:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000010:0.0:1775768284.849842:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6500. 00000100:00000001:2.0:1775768284.849846:0:36:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:0.0:1775768284.849849:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00080000:2.0:1775768284.849850:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5537 00000400:00000200:2.0:1775768284.849858:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768284.849860:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768284.849864:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000400:00000200:0.0:1775768284.849866:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000100:00080000:2.0:1775768284.849867:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000400:00000010:0.0:1775768284.849871:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000040:2.0:1775768284.849872:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768284.849875:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768284.849877:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768284.849880:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:0.0:1775768284.849882:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:2.0:1775768284.849882:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775768284.849885:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:2.0:1775768284.849887:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775768284.849889:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:2.0:1775768284.849891:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:2.0:1775768284.849892:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000200:0.0:1775768284.849896:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfdc00 x1862022294296960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768309 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775768284.849898:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768284.849900:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1775768284.849911:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768284.849912:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768284.849914:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775768284.849917:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00000040:0.0:1775768284.849918:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfdc00 x1862022294296960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768309 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768284.849926:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775768284.849931:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768284.849936:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1775768284.849940:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768284.849941:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768284.849943:0:2399:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:2.0:1775768284.849945:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768284.849948:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775768284.849949:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1775768284.849951:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:2.0:1775768284.849954:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768284.849955:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768284.849959:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4300211005 00000100:00000001:2.0:1775768284.849959:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775768284.849964:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfe300. 00000100:00000010:0.0:1775768284.849966:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942078a3d80. 00000020:00000040:2.0:1775768284.849967:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:2.0:1775768284.849972:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775768284.849973:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768284.849974:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1775768284.849975:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 02000000:00000001:2.0:1775768284.849976:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775768284.849979:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768284.849984:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4300211005 02000000:00000010:2.0:1775768284.849984:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1f400. 00000100:00000001:2.0:1775768284.849990:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768284.849991:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942078a3480. 00000100:00000001:0.0:1775768284.849996:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00100000:2.0:1775768284.849996:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfe300 x1862022294297344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768284.850001:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768284.850007:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768284.850011:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768284.850012:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1775768284.850015:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768284.850020:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768284/real 1775768284] req@ffff994207dfdc00 x1862022294296960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775768284.850021:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c661fef] to pc [ptlrpcd_rcv+-1] req@ffff994207dfe300 x1862022294297344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768284.850040:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768284.850042:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000001:0.0:1775768284.850051:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:2.0:1775768284.850053:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000400:00000001:0.0:1775768284.850054:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768284.850057:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbc268. 00000100:00000001:2.0:1775768284.850059:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:0.0:1775768284.850061:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014ee0 00000100:00080000:2.0:1775768284.850062:0:36:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000010:0.0:1775768284.850065:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014ee0. 00000100:00000001:2.0:1775768284.850066:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:0.0:1775768284.850068:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00080000:2.0:1775768284.850071:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5537 00000100:00000200:0.0:1775768284.850074:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfdc00 x1862022294296960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1775768284.850079:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768284.850085:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775768284.850087:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775768284.850093:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768284.850097:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:0.0:1775768284.850098:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfdc00 x1862022294296960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775768284.850101:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768284.850106:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775768284.850110:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775768284.850113:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000100:00000001:0.0:1775768284.850114:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:2.0:1775768284.850115:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:0.0:1775768284.850117:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:2.0:1775768284.850118:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000100:00000001:0.0:1775768284.850122:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:2.0:1775768284.850123:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000001:0.0:1775768284.850125:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1775768284.850126:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:2.0:1775768284.850132:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00100000:0.0:1775768284.850134:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfdc00 x1862022294296960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768284.850148:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1775768284.850175:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768284.850181:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfdc00 x1862022294296960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768309 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768284.850186:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768284.850189:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768284.850190:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768284.850192:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768284.850194:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000040:2.0:1775768284.850196:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768284.850197:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768284.850200:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00080000:0.0:1775768284.850202:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000020:00000040:2.0:1775768284.850206:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:0.0:1775768284.850210:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00080000:2.0:1775768284.850212:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768284.850215:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1775768284.850216:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768284.850220:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:0.0:1775768284.850224:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfdc00 x1862022294296960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768309 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1775768284.850224:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768284.850230:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768284.850234:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775768284.850236:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:2.0:1775768284.850240:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768284.850241:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768284.850243:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294296960:192.168.202.156@tcp:250:kworker.0 00000100:00000010:2.0:1775768284.850244:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfed80. 00000020:00000040:2.0:1775768284.850248:0:36:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00000100:00000001:0.0:1775768284.850250:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1775768284.850252:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1775768284.850254:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1775768284.850254:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768284.850257:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000040:0.0:1775768284.850259:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfdc00 x1862022294296960/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768309 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1775768284.850260:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775768284.850263:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775768284.850269:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1f800. 00000100:00000001:0.0:1775768284.850270:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768284.850272:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768284.850275:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002a800. 00000100:00000001:2.0:1775768284.850276:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768284.850278:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768284.850280:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768284.850281:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:0.0:1775768284.850284:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00100000:2.0:1775768284.850284:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfed80 x1862022294297472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000010:0.0:1775768284.850287:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1fc00. 02000000:00000001:0.0:1775768284.850291:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768284.850294:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768284.850296:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfdc00. 00000100:00000001:2.0:1775768284.850297:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768284.850298:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768284.850301:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768284.850301:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768284.850306:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb9b9eae] to pc [ptlrpcd_rcv+-1] req@ffff994207dfed80 x1862022294297472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768284.850312:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768284.850316:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768284.850320:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:2.0:1775768284.850320:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5547) 00000100:00000001:0.0:1775768284.850322:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768284.850325:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775768284.850326:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768284.850327:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768284.850331:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dff100 x1862022294297088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768284.850335:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768284.850336:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768284.850338:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768284.850339:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768284.850340:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768284.850342:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294297088:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768284.850344:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768284.850345:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768284.850346:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768284.850347:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768284.850350:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002a800. 02000000:00000001:0.0:1775768284.850351:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768284.850354:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768284.850356:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbc268. 00000400:00000010:0.0:1775768284.850358:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014ee0. 00000100:00000200:0.0:1775768284.850360:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294297088, portal 10 00000100:00000001:0.0:1775768284.850361:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768284.850362:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611848960 : -112966097702656 : ffff994207dff100) 00000100:00000040:0.0:1775768284.850366:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dff100 x1862022294297088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768309 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768284.850370:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768284.850371:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768284.850373:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768284.850374:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294297088, offset 0 00000400:00000200:0.0:1775768284.850376:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768284.850379:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768284.850382:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768284.850384:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768284.850386:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768284.850388:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768284.850389:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768284.850391:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225494:225494:256:4294967295] 192.168.202.156@tcp LPNI seq info [225490:225490:8:4294967295] 00000400:00000200:0.0:1775768284.850393:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768284.850397:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768284.850400:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768284.850402:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6500. 00000800:00000200:0.0:1775768284.850405:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768284.850407:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775768284.850409:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768284.850411:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1775768284.850413:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768284.850414:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6500. 00000400:00000200:0.0:1775768284.850416:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768284.850418:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768284.850420:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768284.850422:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768284.850423:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768284.850425:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dff100 x1862022294297088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768309 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768284.850430:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768284.850431:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dff100 x1862022294297088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768309 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768284.850435:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768284.850436:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768284.850438:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768284.850440:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:0.0:1775768284.850442:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768284.850444:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300209683 00000100:00000001:0.0:1775768284.850445:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300211005 00000100:00000001:0.0:1775768284.850447:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300209683 00000100:00000001:0.0:1775768284.850448:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300211005 00000100:00000001:0.0:1775768284.850449:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768284.850450:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768284.850451:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768284.850453:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768284/real 1775768284] req@ffff994207dff100 x1862022294297088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768284.850458:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768284.850458:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768284.850459:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbc268. 00000400:00000200:0.0:1775768284.850461:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014ee0 00000400:00000010:0.0:1775768284.850462:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014ee0. 00000100:00000001:0.0:1775768284.850463:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768284.850465:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dff100 x1862022294297088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768284.850469:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dff100 x1862022294297088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768284.850473:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768284.850474:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768284.850475:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768284.850475:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768284.850477:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dff100 x1862022294297088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768284.850481:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768284.850482:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dff100 x1862022294297088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768309 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768284.850487:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768284.850489:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768284.850490:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768284.850490:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768284.850491:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768284.850493:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768284.850494:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768284.850496:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768284.850497:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768284.850499:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dff100 x1862022294297088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768309 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768284.850503:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294297088:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768284.850505:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768284.850507:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfc700 x1862022294297216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768284.850511:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768284.850512:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768284.850513:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768284.850514:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768284.850515:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768284.850516:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294297216:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768284.850518:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768284.850519:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768284.850520:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768284.850521:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768284.850523:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220029400. 02000000:00000001:0.0:1775768284.850524:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768284.850526:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768284.850527:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbc268. 00000400:00000010:0.0:1775768284.850529:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014ee0. 00000100:00000200:0.0:1775768284.850531:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294297216, portal 10 00000100:00000001:0.0:1775768284.850532:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768284.850533:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611838208 : -112966097713408 : ffff994207dfc700) 00000100:00000040:0.0:1775768284.850536:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfc700 x1862022294297216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768309 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768284.850540:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768284.850541:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768284.850542:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768284.850543:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294297216, offset 0 00000400:00000200:0.0:1775768284.850546:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768284.850548:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768284.850550:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768284.850551:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768284.850553:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768284.850554:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768284.850556:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768284.850557:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225495:225495:256:4294967295] 192.168.202.156@tcp LPNI seq info [225491:225491:8:4294967295] 00000400:00000200:0.0:1775768284.850559:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768284.850563:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768284.850566:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768284.850567:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6500. 00000800:00000200:0.0:1775768284.850570:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768284.850571:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775768284.850574:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768284.850576:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1775768284.850580:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768284.850581:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6500. 00000400:00000200:0.0:1775768284.850583:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768284.850586:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768284.850589:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768284.850591:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768284.850593:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768284.850595:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfc700 x1862022294297216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768309 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768284.850602:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768284.850605:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfc700 x1862022294297216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768309 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768284.850611:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768284.850613:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768284.850616:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768284.850618:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:0.0:1775768284.850621:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768284.850623:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300209675 00000100:00000001:0.0:1775768284.850625:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300211005 00000100:00000001:0.0:1775768284.850627:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300209675 00000100:00000001:0.0:1775768284.850629:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300211005 00000100:00000001:0.0:1775768284.850631:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768284.850633:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768284.850634:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768284.850637:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768284/real 1775768284] req@ffff994207dfc700 x1862022294297216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768284.850644:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768284.850645:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768284.850646:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbc268. 00000400:00000200:0.0:1775768284.850649:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014ee0 00000400:00000010:0.0:1775768284.850650:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014ee0. 00000100:00000001:0.0:1775768284.850652:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768284.850654:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfc700 x1862022294297216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768284.850662:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfc700 x1862022294297216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768284.850666:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768284.850667:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768284.850668:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768284.850669:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768284.850671:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfc700 x1862022294297216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768284.850675:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768284.850677:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfc700 x1862022294297216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768309 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768284.850680:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768284.850682:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768284.850682:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768284.850683:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768284.850684:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768284.850686:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768284.850687:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768284.850689:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768284.850690:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768284.850692:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfc700 x1862022294297216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768309 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768284.850696:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294297216:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768284.850698:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768284.850700:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfe300 x1862022294297344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768284.850704:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768284.850704:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768284.850706:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768284.850707:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768284.850709:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768284.850710:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294297344:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768284.850712:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768284.850713:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768284.850714:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768284.850715:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768284.850717:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220029800. 02000000:00000001:0.0:1775768284.850718:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768284.850721:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768284.850722:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbc268. 00000400:00000010:0.0:1775768284.850724:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014ee0. 00000100:00000200:0.0:1775768284.850726:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294297344, portal 4 00000100:00000001:0.0:1775768284.850727:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768284.850728:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611845376 : -112966097706240 : ffff994207dfe300) 00000100:00000040:0.0:1775768284.850731:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfe300 x1862022294297344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768309 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768284.850735:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768284.850737:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768284.850738:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768284.850739:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294297344, offset 0 00000400:00000200:0.0:1775768284.850741:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768284.850744:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768284.850746:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768284.850748:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768284.850749:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768284.850751:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768284.850752:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768284.850754:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225496:225496:256:4294967295] 192.168.202.156@tcp LPNI seq info [225492:225492:8:4294967295] 00000400:00000200:0.0:1775768284.850758:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768284.850764:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768284.850768:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768284.850770:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6500. 00000800:00000200:0.0:1775768284.850774:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768284.850776:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775768284.850780:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768284.850782:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1775768284.850785:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768284.850787:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6500. 00000400:00000200:0.0:1775768284.850790:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768284.850793:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768284.850796:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768284.850797:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768284.850800:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768284.850803:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfe300 x1862022294297344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768309 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768284.850810:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768284.850813:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfe300 x1862022294297344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768309 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768284.850819:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768284.850821:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768284.850824:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768284.850827:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:0.0:1775768284.850830:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768284.850831:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300209698 00000100:00000001:0.0:1775768284.850834:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300211005 00000100:00000001:0.0:1775768284.850835:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300209698 00000100:00000001:0.0:1775768284.850837:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300211005 00000100:00000001:0.0:1775768284.850839:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768284.850841:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768284.850843:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768284.850847:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768284/real 1775768284] req@ffff994207dfe300 x1862022294297344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768284.850854:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768284.850855:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768284.850857:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbc268. 00000400:00000200:0.0:1775768284.850859:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014ee0 00000400:00000010:0.0:1775768284.850860:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014ee0. 00000100:00000001:0.0:1775768284.850862:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768284.850865:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfe300 x1862022294297344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768284.850870:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfe300 x1862022294297344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768284.850873:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768284.850874:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768284.850875:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768284.850876:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768284.850877:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfe300 x1862022294297344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768284.850881:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768284.850883:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfe300 x1862022294297344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768309 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768284.850887:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768284.850888:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768284.850889:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768284.850890:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768284.850891:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768284.850893:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768284.850894:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768284.850896:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768284.850897:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768284.850899:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfe300 x1862022294297344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768309 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768284.850904:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294297344:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768284.850907:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768284.850910:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfed80 x1862022294297472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768284.850916:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768284.850918:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768284.850920:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768284.850921:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768284.850923:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768284.850925:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294297472:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768284.850928:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768284.850930:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768284.850931:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768284.850933:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768284.850936:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002bc00. 02000000:00000001:0.0:1775768284.850937:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768284.850940:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768284.850943:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbc268. 00000400:00000010:0.0:1775768284.850945:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014ee0. 00000100:00000200:0.0:1775768284.850948:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294297472, portal 4 00000100:00000001:0.0:1775768284.850949:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768284.850950:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611848064 : -112966097703552 : ffff994207dfed80) 00000100:00000040:0.0:1775768284.850955:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfed80 x1862022294297472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768309 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768284.850960:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768284.850962:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768284.850965:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768284.850967:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294297472, offset 0 00000400:00000200:0.0:1775768284.850971:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768284.850973:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768284.850976:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768284.850977:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768284.850979:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768284.850981:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768284.850983:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768284.850986:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225497:225497:256:4294967295] 192.168.202.156@tcp LPNI seq info [225493:225493:8:4294967295] 00000400:00000200:0.0:1775768284.850990:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768284.850995:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768284.850999:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768284.851002:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6500. 00000800:00000200:0.0:1775768284.851005:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768284.851008:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775768284.851011:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768284.851013:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1775768284.851017:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768284.851018:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6500. 00000400:00000200:0.0:1775768284.851021:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768284.851023:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768284.851026:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768284.851027:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768284.851029:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768284.851032:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207dfed80 x1862022294297472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768309 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768284.851038:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768284.851040:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfed80 x1862022294297472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768309 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768284.851045:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768284.851046:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768284.851049:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768284.851051:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:0.0:1775768284.851054:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768284.851056:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300209701 00000100:00000001:0.0:1775768284.851058:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300211005 00000100:00000001:0.0:1775768284.851059:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300209701 00000100:00000001:0.0:1775768284.851061:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300211005 00000100:00000001:0.0:1775768284.851063:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768284.851065:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768284.851067:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768284.851070:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768284/real 1775768284] req@ffff994207dfed80 x1862022294297472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768284.851075:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768284.851077:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768284.851078:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbc268. 00000400:00000200:0.0:1775768284.851080:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014ee0 00000400:00000010:0.0:1775768284.851081:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014ee0. 00000100:00000001:0.0:1775768284.851083:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768284.851085:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfed80 x1862022294297472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768284.851091:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfed80 x1862022294297472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768284.851096:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768284.851097:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768284.851099:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768284.851100:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768284.851102:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfed80 x1862022294297472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768284.851108:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768284.851111:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfed80 x1862022294297472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768309 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768284.851116:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768284.851119:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768284.851120:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768284.851121:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768284.851123:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768284.851126:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768284.851128:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768284.851131:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768284.851133:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768284.851136:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfed80 x1862022294297472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768309 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768284.851168:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294297472:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768284.851171:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768284.851173:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768284.851175:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dff100 x1862022294297088/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768309 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768284.851180:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768284.851181:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768284.851183:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002a800. 02000000:00000001:0.0:1775768284.851185:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768284.851187:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768284.851188:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:0.0:1775768284.851190:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768284.851192:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1ec00. 02000000:00000001:0.0:1775768284.851196:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768284.851197:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768284.851198:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dff100. 00000100:00000001:0.0:1775768284.851201:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768284.851202:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768284.851203:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768284.851206:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfc700 x1862022294297216/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768309 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768284.851217:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768284.851218:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768284.851220:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220029400. 02000000:00000001:0.0:1775768284.851222:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768284.851223:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768284.851224:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:0.0:1775768284.851226:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768284.851227:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1f000. 02000000:00000001:0.0:1775768284.851229:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768284.851230:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768284.851231:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfc700. 00000100:00000001:0.0:1775768284.851233:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768284.851234:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768284.851235:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768284.851238:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfe300 x1862022294297344/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768309 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768284.851245:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768284.851246:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768284.851247:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220029800. 02000000:00000001:0.0:1775768284.851249:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768284.851250:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768284.851251:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:0.0:1775768284.851252:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768284.851253:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1f400. 02000000:00000001:0.0:1775768284.851255:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768284.851256:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768284.851257:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfe300. 00000100:00000001:0.0:1775768284.851259:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768284.851260:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768284.851262:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768284.851264:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfed80 x1862022294297472/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768309 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768284.851268:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768284.851269:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768284.851270:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002bc00. 02000000:00000001:0.0:1775768284.851271:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768284.851272:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768284.851273:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:0.0:1775768284.851274:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768284.851275:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1f800. 02000000:00000001:0.0:1775768284.851276:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768284.851276:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768284.851277:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfed80. 00000100:00000001:0.0:1775768284.851278:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768284.851278:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768284.851279:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768284.851282:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768284.851283:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768284.851284:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775768284.851285:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768284.913264:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768284.913277:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768284.913284:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768284.913285:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768284.913289:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768284.913290:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768284.913294:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768284.913295:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768284.913299:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768284.913300:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768284.913357:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768284.913361:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768284.913364:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768284.977322:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768284.977329:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768284.977331:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768284.977336:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768284.977340:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768284.977343:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768284.977346:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768284.977349:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768284.977351:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768285.873518:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768285.873526:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768285.873528:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768285.873534:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768285.873539:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768285.873543:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768285.873546:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768285.873548:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768285.873551:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768285.873805:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768285.873815:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768285.873817:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768285.873820:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768285.873823:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768285.873825:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768285.873828:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768285.873829:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768285.873832:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768285.874000:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768285.874004:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768285.874006:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768285.874010:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768285.874012:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768285.874014:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768285.874016:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768285.874018:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768285.874020:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768285.937301:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768285.937307:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768285.937311:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768285.937312:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768285.937316:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768285.937317:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768285.937320:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768285.937321:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768285.937324:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768285.937324:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768285.937345:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768285.937349:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768285.937352:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768286.001282:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768286.001289:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768286.001291:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768286.001296:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768286.001300:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768286.001304:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768286.001306:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768286.001308:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768286.001310:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768286.193531:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4300211005, at 4300211008 00000100:00000010:1.0:1775768286.193554:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3480. 00000100:00000040:0.0:1775768286.194025:0:15036:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4300211005, at 4300211008 00000100:00000010:0.0:1775768286.194033:0:15036:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3d80. 00000100:00000001:1.0:1775768286.897341:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768286.897350:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768286.897352:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768286.897360:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768286.897376:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768286.897382:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768286.897385:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768286.897388:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768286.897392:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768286.897540:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768286.897545:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768286.897546:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768286.897549:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768286.897551:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768286.897553:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768286.897555:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768286.897557:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768286.897558:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768286.897782:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768286.897788:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768286.897790:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768286.898529:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768286.898533:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768286.898537:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768286.898539:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768286.898541:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768286.898544:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768286.961266:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768286.961272:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768286.961277:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768286.961279:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768286.961283:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768286.961285:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768286.961287:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768286.961289:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768286.961292:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768286.961294:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768286.961371:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768286.961375:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768286.961379:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768287.025299:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768287.025309:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768287.025312:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768287.025320:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768287.025328:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768287.025334:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768287.025337:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768287.025340:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768287.025343:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768287.921245:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768287.921250:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768287.921252:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768287.921257:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768287.921260:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768287.921264:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768287.921266:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768287.921268:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768287.921270:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768287.921332:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768287.921338:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768287.921341:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768287.921346:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768287.921350:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768287.921352:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768287.921354:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768287.921358:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768287.921358:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768287.921361:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768287.921362:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768287.921366:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768287.921368:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768287.921370:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768287.921373:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768287.921376:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768287.921378:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768287.921380:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768287.985318:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768287.985325:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768287.985332:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768287.985335:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768287.985338:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768287.985340:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768287.985343:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768287.985345:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768287.985347:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768287.985349:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768287.985352:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768287.985352:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768287.985353:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768287.985354:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768287.985355:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768287.985356:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768287.985358:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768287.985359:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768287.985361:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768287.985363:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768287.985363:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768287.985365:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768287.985365:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768287.985366:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768287.985367:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768287.985368:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768287.985371:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768287.985371:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768287.985372:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768287.985373:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768287.985374:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768287.985374:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768287.985375:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768287.985376:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768287.985394:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768287.985396:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768287.985398:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768288.049375:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768288.049381:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768288.049382:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768288.049388:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768288.049391:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768288.049395:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768288.049397:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768288.049400:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768288.049402:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768288.945244:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768288.945247:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768288.945249:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768288.945251:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768288.945252:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768288.945256:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768288.945256:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:3.0:1775768288.945257:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775768288.945262:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768288.945262:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:0.0:1775768288.945263:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768288.945266:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768288.945267:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768288.945268:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:2.0:1775768288.945271:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775768288.945271:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768288.945272:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768288.945274:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768288.945275:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768288.945279:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768288.945279:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768288.945279:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768288.945283:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768288.945286:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768288.945290:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768288.945293:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768288.945295:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768289.009239:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768289.009245:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768289.009252:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768289.009262:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768289.009265:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768289.009266:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768289.009270:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768289.009272:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768289.009275:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768289.009276:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768289.009327:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768289.009331:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768289.009334:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768289.073261:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768289.073267:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768289.073269:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768289.073274:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768289.073277:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768289.073281:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768289.073284:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768289.073286:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768289.073288:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768289.969264:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768289.969264:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768289.969264:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768289.969271:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768289.969271:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768289.969271:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768289.969274:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768289.969275:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768289.969275:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768289.969281:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775768289.969281:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775768289.969281:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00080000:1.0:1775768289.969283:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1775768289.969286:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768289.969287:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768289.969290:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768289.969292:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768289.969292:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768289.969293:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1775768289.969293:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775768289.969298:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768289.969298:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1775768289.969302:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1775768289.969303:0:36:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768289.969305:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768289.969305:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768289.969308:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775768289.969308:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768289.969309:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768289.969310:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768289.969313:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1775768289.969317:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5542 00000100:00000001:3.0:1775768289.969317:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768289.969328:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768289.969336:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775768289.969339:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775768289.969344:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768289.969348:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768289.969350:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768289.969353:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775768289.969358:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775768289.969360:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:1.0:1775768289.969363:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775768289.969366:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:1.0:1775768289.969369:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:1.0:1775768289.969371:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:1.0:1775768289.969374:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768289.969376:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768289.969378:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768289.969380:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775768289.969382:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:1.0:1775768289.969385:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775768289.969388:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768289.969390:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775768289.969393:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768289.969396:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768289.969398:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775768289.969400:0:36:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775768289.969409:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207888e00. 00000020:00000040:1.0:1775768289.969413:0:36:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:1.0:1775768289.969416:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768289.969418:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768289.969420:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775768289.969423:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775768289.969427:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775768289.969435:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543f000. 00000100:00000001:1.0:1775768289.969441:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768289.969449:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207888e00 x1862022294297600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775768289.969463:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775768289.969467:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768289.969471:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001812f966] to pc [ptlrpcd_rcv+-1] req@ffff994207888e00 x1862022294297600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768289.969490:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768289.969493:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768289.969496:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775768289.969498:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775768289.969499:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768289.969502:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775768289.969505:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5542 00000400:00000200:1.0:1775768289.969510:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768289.969513:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775768289.969516:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775768289.969519:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768289.969520:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768289.969522:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1775768289.969524:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1775768289.969526:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768289.969527:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1775768289.969529:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775768289.969532:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775768289.969534:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000100:00000040:0.0:1775768289.969536:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207888e00 x1862022294297600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1775768289.969536:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775768289.969538:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:1.0:1775768289.969542:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:1.0:1775768289.969547:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000001:0.0:1775768289.969553:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000040:1.0:1775768289.969553:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768289.969556:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768289.969557:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768289.969561:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000001:0.0:1775768289.969562:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1775768289.969563:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 02000000:00000001:0.0:1775768289.969566:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:1.0:1775768289.969567:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 02000000:00000001:0.0:1775768289.969570:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768289.969572:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00100000:0.0:1775768289.969577:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207888e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294297600:192.168.202.156@tcp:250:kworker.0 00000100:00000001:1.0:1775768289.969577:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768289.969580:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1775768289.969585:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768289.969586:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768289.969589:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768289.969590:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768289.969592:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768289.969593:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:0.0:1775768289.969597:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:1.0:1775768289.969598:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 02000000:00000010:0.0:1775768289.969602:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002bc00. 00000008:00000001:1.0:1775768289.969602:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768289.969605:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768289.969606:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775768289.969610:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207889f80. 00000400:00000200:0.0:1775768289.969614:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000040:1.0:1775768289.969614:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000400:00000010:0.0:1775768289.969618:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbc268. 02000000:00000001:1.0:1775768289.969618:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768289.969621:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768289.969623:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000010:0.0:1775768289.969625:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014ee0. 02000000:00000001:1.0:1775768289.969625:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775768289.969627:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775768289.969632:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543e800. 00000100:00000200:0.0:1775768289.969634:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294297600, portal 25 00000100:00000001:1.0:1775768289.969636:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768289.969638:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768289.969640:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606121984 : -112966103429632 : ffff994207888e00) 00000100:00100000:1.0:1775768289.969643:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207889f80 x1862022294297728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:0.0:1775768289.969650:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207888e00 x1862022294297600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768314 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768289.969656:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775768289.969660:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768289.969663:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768289.969668:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ed2274f4] to pc [ptlrpcd_rcv+-1] req@ffff994207889f80 x1862022294297728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775768289.969669:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768289.969673:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000001:1.0:1775768289.969681:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768289.969683:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:0.0:1775768289.969686:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294297600, offset 0 00000100:00080000:1.0:1775768289.969687:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775768289.969690:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775768289.969693:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:0.0:1775768289.969697:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00000001:1.0:1775768289.969697:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775768289.969702:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5542 00000400:00000200:1.0:1775768289.969705:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768289.969707:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775768289.969709:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775768289.969711:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768289.969712:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1775768289.969715:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000040:1.0:1775768289.969715:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768289.969719:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775768289.969720:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775768289.969721:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:1.0:1775768289.969722:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:0.0:1775768289.969725:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000020:00000040:1.0:1775768289.969725:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:1.0:1775768289.969727:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:1.0:1775768289.969729:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000400:00000200:0.0:1775768289.969732:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:1.0:1775768289.969732:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768289.969736:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768289.969737:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768289.969738:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000200:0.0:1775768289.969739:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000020:00000040:1.0:1775768289.969740:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:1.0:1775768289.969744:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000400:00000200:0.0:1775768289.969747:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000100:00000001:1.0:1775768289.969748:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768289.969751:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:0.0:1775768289.969753:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 02000000:00000001:1.0:1775768289.969754:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768289.969756:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768289.969759:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225498:225498:256:4294967295] 192.168.202.156@tcp LPNI seq info [225494:225494:8:4294967295] 00000100:00000001:1.0:1775768289.969759:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775768289.969762:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775768289.969766:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768289.969767:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768289.969769:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775768289.969774:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff99420788b100. 00000020:00000040:1.0:1775768289.969777:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 02000000:00000001:1.0:1775768289.969779:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768289.969781:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768289.969784:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:0.0:1775768289.969787:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 02000000:00000001:1.0:1775768289.969787:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775768289.969790:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775768289.969795:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543d400. 00000800:00000200:0.0:1775768289.969796:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00000001:1.0:1775768289.969800:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1775768289.969802:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6500. 00000100:00100000:1.0:1775768289.969807:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff99420788b100 x1862022294297856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:0.0:1775768289.969813:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768289.969818:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000100:00000001:1.0:1775768289.969819:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775768289.969822:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775768289.969824:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000040:1.0:1775768289.969828:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e26cf977] to pc [ptlrpcd_rcv+-1] req@ffff99420788b100 x1862022294297856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.1:1775768289.969828:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1775768289.969837:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000100:00000001:1.0:1775768289.969839:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1775768289.969840:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6500. 00000100:00080000:1.0:1775768289.969841:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768289.969844:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000400:00000200:0.0:1775768289.969845:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:1.0:1775768289.969845:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775768289.969846:0:36:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768289.969849:0:36:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:0.0:1775768289.969850:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000100:00080000:1.0:1775768289.969853:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5542 00000400:00000200:0.0:1775768289.969856:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000200:1.0:1775768289.969857:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768289.969860:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000400:00000200:1.0:1775768289.969861:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775768289.969864:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1775768289.969865:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:1.0:1775768289.969867:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768289.969869:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:0.0:1775768289.969871:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207888e00 x1862022294297600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768314 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775768289.969872:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768289.969884:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:0.0:1775768289.969891:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000020:00000001:1.0:1775768289.969891:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775768289.969893:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000100:00000040:0.0:1775768289.969896:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207888e00 x1862022294297600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768314 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1775768289.969896:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775768289.969899:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:1.0:1775768289.969904:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:0.0:1775768289.969905:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768289.969908:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:1.0:1775768289.969910:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:0.0:1775768289.969913:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768289.969915:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000200:0.0:1775768289.969919:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768289.969919:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1775768289.969923:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4300216125 00000100:00000040:1.0:1775768289.969924:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768289.969927:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000010:0.0:1775768289.969928:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942078a3d80. 00000020:00000040:1.0:1775768289.969929:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00000001:0.0:1775768289.969934:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775768289.969937:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4300216125 00000100:00000010:0.0:1775768289.969941:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942078a3720. 00000100:00080000:1.0:1775768289.969942:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768289.969944:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775768289.969948:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768289.969948:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768289.969951:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768289.969952:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1775768289.969954:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768289.969958:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768289/real 1775768289] req@ffff994207888e00 x1862022294297600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1775768289.969958:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768289.969963:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768289.969966:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775768289.969969:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:1.0:1775768289.969974:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1775768289.969977:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1775768289.969977:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1775768289.969980:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000010:1.0:1775768289.969982:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207889880. 00000400:00000010:0.0:1775768289.969983:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbc268. 00000020:00000040:1.0:1775768289.969985:0:36:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00000400:00000200:0.0:1775768289.969987:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014ee0 02000000:00000001:1.0:1775768289.969989:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000010:0.0:1775768289.969991:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014ee0. 02000000:00000001:1.0:1775768289.969991:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768289.969994:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1775768289.969995:0:2399:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:1.0:1775768289.969996:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775768289.969999:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1775768289.970001:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207888e00 x1862022294297600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1775768289.970004:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543f800. 00000100:00000001:1.0:1775768289.970009:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1775768289.970018:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207888e00 x1862022294297600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768289.970018:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207889880 x1862022294297984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768289.970041:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768289.970042:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768289.970044:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768289.970045:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768289.970048:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00100000:0.0:1775768289.970052:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207888e00 x1862022294297600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768289.970052:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768289.970059:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f11856b8] to pc [ptlrpcd_rcv+-1] req@ffff994207889880 x1862022294297984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768289.970070:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768289.970076:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768289.970080:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775768289.970081:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207888e00 x1862022294297600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768314 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768289.970086:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775768289.970100:0:36:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775768289.970102:0:36:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768289.970107:0:36:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:0.0:1775768289.970109:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775768289.970111:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5542 00000100:00000001:0.0:1775768289.970114:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768289.970117:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000400:00000200:1.0:1775768289.970118:0:36:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768289.970120:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1775768289.970123:0:36:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:0.0:1775768289.970124:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00080000:1.0:1775768289.970126:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1775768289.970127:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768289.970132:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000040:1.0:1775768289.970132:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768289.970136:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768289.970140:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00080000:0.0:1775768289.970141:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:1.0:1775768289.970142:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775768289.970145:0:36:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:0.0:1775768289.970146:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000040:1.0:1775768289.970146:0:36:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:1.0:1775768289.970148:0:36:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775768289.970150:0:36:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:1.0:1775768289.970152:0:36:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:0.0:1775768289.970176:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207888e00 x1862022294297600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768314 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1775768289.970181:0:36:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:1.0:1775768289.970187:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768289.970189:0:36:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:0.0:1775768289.970190:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207888e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294297600:192.168.202.156@tcp:250:kworker.0 00000100:00000040:1.0:1775768289.970192:0:36:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768289.970195:0:36:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775768289.970198:0:36:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00000001:0.0:1775768289.970199:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1775768289.970201:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768289.970203:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1775768289.970204:0:36:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768289.970207:0:36:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775768289.970209:0:36:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1775768289.970210:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207888e00 x1862022294297600/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768314 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1775768289.970212:0:36:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768289.970215:0:36:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775768289.970217:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:1.0:1775768289.970221:0:36:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768289.970224:0:36:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768289.970227:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 00000100:00000010:1.0:1775768289.970228:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207888700. 02000000:00000001:0.0:1775768289.970230:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000040:1.0:1775768289.970231:0:36:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:1.0:1775768289.970234:0:36:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000010:0.0:1775768289.970235:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002bc00. 02000000:00000001:1.0:1775768289.970235:0:36:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768289.970236:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775768289.970239:0:36:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775768289.970240:0:36:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768289.970241:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000010:1.0:1775768289.970243:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543fc00. 00000020:00000001:0.0:1775768289.970244:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768289.970247:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000100:00000001:1.0:1775768289.970247:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768289.970251:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00100000:1.0:1775768289.970254:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207888700 x1862022294298112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000010:0.0:1775768289.970255:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543f000. 00000100:00000001:1.0:1775768289.970261:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:0.0:1775768289.970262:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:1.0:1775768289.970263:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768289.970265:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000040:1.0:1775768289.970267:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009f2ba301] to pc [ptlrpcd_rcv+-1] req@ffff994207888700 x1862022294298112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1775768289.970268:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207888e00. 00000100:00000001:0.0:1775768289.970274:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775768289.970275:0:36:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768289.970276:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768289.970278:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5552) 00000100:00000001:0.0:1775768289.970279:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768289.970287:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768289.970289:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768289.970291:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775768289.970294:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768289.970295:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768289.970300:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207889f80 x1862022294297728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768289.970308:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768289.970309:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768289.970312:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768289.970314:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768289.970316:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768289.970319:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207889f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294297728:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768289.970322:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768289.970323:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768289.970325:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768289.970327:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768289.970329:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002bc00. 02000000:00000001:0.0:1775768289.970331:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768289.970335:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768289.970337:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbc268. 00000400:00000010:0.0:1775768289.970341:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014ee0. 00000100:00000200:0.0:1775768289.970343:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294297728, portal 10 00000100:00000001:0.0:1775768289.970345:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768289.970346:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606126464 : -112966103425152 : ffff994207889f80) 00000100:00000040:0.0:1775768289.970352:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207889f80 x1862022294297728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768314 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768289.970358:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768289.970360:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768289.970363:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768289.970365:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294297728, offset 0 00000400:00000200:0.0:1775768289.970369:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768289.970373:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768289.970377:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768289.970380:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768289.970382:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768289.970384:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768289.970387:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768289.970390:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225499:225499:256:4294967295] 192.168.202.156@tcp LPNI seq info [225495:225495:8:4294967295] 00000400:00000200:0.0:1775768289.970394:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768289.970400:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768289.970405:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768289.970408:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6500. 00000800:00000200:0.0:1775768289.970412:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768289.970414:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775768289.970419:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768289.970421:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1775768289.970425:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768289.970426:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6500. 00000400:00000200:0.0:1775768289.970429:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768289.970432:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768289.970435:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768289.970437:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768289.970439:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768289.970442:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207889f80 x1862022294297728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768314 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768289.970450:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768289.970453:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207889f80 x1862022294297728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768314 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768289.970460:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768289.970461:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768289.970464:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768289.970467:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:0.0:1775768289.970470:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768289.970473:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300214803 00000100:00000001:0.0:1775768289.970475:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300216125 00000100:00000001:0.0:1775768289.970477:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300214803 00000100:00000001:0.0:1775768289.970479:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300216125 00000100:00000001:0.0:1775768289.970481:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768289.970483:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768289.970485:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768289.970488:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768289/real 1775768289] req@ffff994207889f80 x1862022294297728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768289.970496:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768289.970498:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768289.970499:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbc268. 00000400:00000200:0.0:1775768289.970501:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014ee0 00000400:00000010:0.0:1775768289.970503:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014ee0. 00000100:00000001:0.0:1775768289.970505:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768289.970508:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207889f80 x1862022294297728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768289.970517:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207889f80 x1862022294297728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768289.970523:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768289.970524:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768289.970526:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768289.970527:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768289.970530:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207889f80 x1862022294297728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768289.970536:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768289.970540:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207889f80 x1862022294297728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768314 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768289.970546:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768289.970548:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768289.970550:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768289.970551:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768289.970552:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768289.970555:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768289.970557:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768289.970560:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768289.970562:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768289.970566:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207889f80 x1862022294297728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768314 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768289.970574:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207889f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294297728:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768289.970577:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768289.970580:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff99420788b100 x1862022294297856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768289.970586:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768289.970587:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768289.970589:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768289.970591:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768289.970593:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768289.970595:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff99420788b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294297856:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768289.970598:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768289.970600:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768289.970601:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768289.970603:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768289.970606:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220029800. 02000000:00000001:0.0:1775768289.970608:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768289.970611:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768289.970613:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbc268. 00000400:00000010:0.0:1775768289.970616:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014ee0. 00000100:00000200:0.0:1775768289.970619:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294297856, portal 10 00000100:00000001:0.0:1775768289.970621:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768289.970622:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606130944 : -112966103420672 : ffff99420788b100) 00000100:00000040:0.0:1775768289.970627:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff99420788b100 x1862022294297856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768314 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768289.970633:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768289.970635:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768289.970636:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768289.970638:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294297856, offset 0 00000400:00000200:0.0:1775768289.970642:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768289.970645:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768289.970649:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768289.970651:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768289.970653:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768289.970655:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768289.970657:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768289.970660:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225500:225500:256:4294967295] 192.168.202.156@tcp LPNI seq info [225496:225496:8:4294967295] 00000400:00000200:0.0:1775768289.970664:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768289.970669:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768289.970673:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768289.970677:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6500. 00000800:00000200:0.0:1775768289.970680:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768289.970683:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775768289.970686:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768289.970689:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1775768289.970692:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768289.970694:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6500. 00000400:00000200:0.0:1775768289.970696:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768289.970700:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768289.970703:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768289.970704:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768289.970707:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768289.970711:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff99420788b100 x1862022294297856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768314 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768289.970718:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768289.970721:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff99420788b100 x1862022294297856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768314 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768289.970727:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768289.970728:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768289.970731:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768289.970733:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:0.0:1775768289.970736:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768289.970738:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300214795 00000100:00000001:0.0:1775768289.970740:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300216125 00000100:00000001:0.0:1775768289.970742:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300214795 00000100:00000001:0.0:1775768289.970744:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300216125 00000100:00000001:0.0:1775768289.970746:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768289.970748:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768289.970749:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768289.970752:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768289/real 1775768289] req@ffff99420788b100 x1862022294297856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768289.970759:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768289.970760:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768289.970762:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbc268. 00000400:00000200:0.0:1775768289.970764:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014ee0 00000400:00000010:0.0:1775768289.970766:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014ee0. 00000100:00000001:0.0:1775768289.970767:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768289.970769:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff99420788b100 x1862022294297856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768289.970774:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff99420788b100 x1862022294297856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768289.970778:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768289.970779:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768289.970780:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768289.970781:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768289.970783:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff99420788b100 x1862022294297856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768289.970786:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768289.970788:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff99420788b100 x1862022294297856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768314 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768289.970792:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768289.970794:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768289.970795:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768289.970795:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768289.970796:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768289.970798:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768289.970799:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768289.970800:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768289.970802:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768289.970804:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff99420788b100 x1862022294297856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768314 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768289.970808:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff99420788b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294297856:192.168.202.156@tcp:38:kworker.0 00000100:00000001:0.0:1775768289.970810:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768289.970812:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207889880 x1862022294297984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768289.970816:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768289.970816:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768289.970818:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768289.970819:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768289.970820:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768289.970822:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207889880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294297984:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768289.970824:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768289.970825:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768289.970826:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768289.970827:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768289.970829:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994220029400. 02000000:00000001:0.0:1775768289.970830:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768289.970833:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768289.970834:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbc268. 00000400:00000010:0.0:1775768289.970836:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014ee0. 00000100:00000200:0.0:1775768289.970838:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294297984, portal 4 00000100:00000001:0.0:1775768289.970839:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768289.970840:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606124672 : -112966103426944 : ffff994207889880) 00000100:00000040:0.0:1775768289.970843:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207889880 x1862022294297984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768314 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768289.970846:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768289.970848:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768289.970849:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768289.970850:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294297984, offset 0 00000400:00000200:0.0:1775768289.970852:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768289.970855:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768289.970857:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768289.970858:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768289.970860:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768289.970861:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768289.970862:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768289.970864:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225501:225501:256:4294967295] 192.168.202.156@tcp LPNI seq info [225497:225497:8:4294967295] 00000400:00000200:0.0:1775768289.970867:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768289.970871:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768289.970874:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768289.970875:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6500. 00000800:00000200:0.0:1775768289.970878:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768289.970879:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775768289.970883:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768289.970886:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1775768289.970889:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768289.970890:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6500. 00000400:00000200:0.0:1775768289.970893:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768289.970896:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768289.970898:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768289.970900:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768289.970902:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768289.970905:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207889880 x1862022294297984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768314 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768289.970912:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768289.970915:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207889880 x1862022294297984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768314 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768289.970922:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768289.970923:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768289.970926:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768289.970929:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:0.0:1775768289.970932:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768289.970934:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300214818 00000100:00000001:0.0:1775768289.970936:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300216125 00000100:00000001:0.0:1775768289.970938:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300214818 00000100:00000001:0.0:1775768289.970940:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300216125 00000100:00000001:0.0:1775768289.970942:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768289.970944:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768289.970945:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768289.970948:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768289/real 1775768289] req@ffff994207889880 x1862022294297984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768289.970954:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768289.970954:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768289.970955:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbc268. 00000400:00000200:0.0:1775768289.970957:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014ee0 00000400:00000010:0.0:1775768289.970958:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014ee0. 00000100:00000001:0.0:1775768289.970959:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768289.970961:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207889880 x1862022294297984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768289.970969:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207889880 x1862022294297984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768289.970975:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768289.970976:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768289.970978:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768289.970979:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768289.970982:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207889880 x1862022294297984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768289.970989:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768289.970993:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207889880 x1862022294297984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768314 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768289.970999:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768289.971001:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768289.971003:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768289.971004:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768289.971006:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768289.971009:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768289.971011:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768289.971014:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768289.971015:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768289.971018:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207889880 x1862022294297984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768314 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768289.971023:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207889880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294297984:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768289.971025:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775768289.971026:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207888700 x1862022294298112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768289.971030:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768289.971031:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768289.971032:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775768289.971033:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768289.971034:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768289.971036:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207888700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294298112:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768289.971038:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768289.971039:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768289.971039:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768289.971040:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775768289.971043:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99422002a800. 02000000:00000001:0.0:1775768289.971044:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768289.971046:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:0.0:1775768289.971047:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994208bbc268. 00000400:00000010:0.0:1775768289.971049:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014ee0. 00000100:00000200:0.0:1775768289.971051:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294298112, portal 4 00000100:00000001:0.0:1775768289.971052:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775768289.971053:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606120192 : -112966103431424 : ffff994207888700) 00000100:00000040:0.0:1775768289.971056:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207888700 x1862022294298112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768314 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768289.971060:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775768289.971061:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:0.0:1775768289.971063:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994205014dd0. 00000100:00000200:0.0:1775768289.971064:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294298112, offset 0 00000400:00000200:0.0:1775768289.971066:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768289.971068:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768289.971071:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1775768289.971072:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775768289.971074:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:0.0:1775768289.971075:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775768289.971077:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:0.0:1775768289.971078:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225502:225502:256:4294967295] 192.168.202.156@tcp LPNI seq info [225498:225498:8:4294967295] 00000400:00000200:0.0:1775768289.971081:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768289.971084:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:0.0:1775768289.971087:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768289.971100:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9942068c6500. 00000800:00000200:0.0:1775768289.971103:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.0:1775768289.971105:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775768289.971107:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:0.1:1775768289.971109:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1775768289.971111:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:0.0:1775768289.971112:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9942068c6500. 00000400:00000200:0.0:1775768289.971114:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775768289.971116:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775768289.971118:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014dd0 00000400:00000010:0.0:1775768289.971119:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014dd0. 00000100:00000001:0.0:1775768289.971120:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775768289.971122:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207888700 x1862022294298112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768314 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768289.971127:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768289.971128:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207888700 x1862022294298112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768314 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768289.971132:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768289.971133:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768289.971135:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768289.971136:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:0.0:1775768289.971138:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768289.971139:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300214821 00000100:00000001:0.0:1775768289.971141:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300216125 00000100:00000001:0.0:1775768289.971142:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300214821 00000100:00000001:0.0:1775768289.971143:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300216125 00000100:00000001:0.0:1775768289.971144:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768289.971146:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768289.971147:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775768289.971148:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768289/real 1775768289] req@ffff994207888700 x1862022294298112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775768289.971152:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775768289.971153:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775768289.971171:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994208bbc268. 00000400:00000200:0.0:1775768289.971172:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994205014ee0 00000400:00000010:0.0:1775768289.971173:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994205014ee0. 00000100:00000001:0.0:1775768289.971174:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775768289.971176:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207888700 x1862022294298112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775768289.971181:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207888700 x1862022294298112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768289.971185:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775768289.971186:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768289.971187:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768289.971188:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768289.971189:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207888700 x1862022294298112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768289.971193:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768289.971195:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207888700 x1862022294298112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768314 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768289.971199:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768289.971200:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768289.971201:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768289.971202:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768289.971203:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775768289.971205:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775768289.971207:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768289.971208:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1775768289.971209:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775768289.971211:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207888700 x1862022294298112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768314 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768289.971215:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207888700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294298112:192.168.202.156@tcp:8:kworker.0 00000100:00000001:0.0:1775768289.971217:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768289.971219:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768289.971220:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207889f80 x1862022294297728/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768314 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768289.971224:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768289.971225:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768289.971226:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002bc00. 02000000:00000001:0.0:1775768289.971227:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768289.971228:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768289.971229:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:0.0:1775768289.971230:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768289.971231:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543e800. 02000000:00000001:0.0:1775768289.971232:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768289.971233:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768289.971234:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207889f80. 00000100:00000001:0.0:1775768289.971235:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768289.971236:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768289.971237:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768289.971239:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff99420788b100 x1862022294297856/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768314 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768289.971246:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768289.971247:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768289.971248:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220029800. 02000000:00000001:0.0:1775768289.971249:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768289.971249:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768289.971250:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:0.0:1775768289.971251:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768289.971252:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543d400. 02000000:00000001:0.0:1775768289.971253:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768289.971253:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768289.971254:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff99420788b100. 00000100:00000001:0.0:1775768289.971255:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768289.971256:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768289.971257:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768289.971258:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207889880 x1862022294297984/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768314 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768289.971261:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768289.971262:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768289.971262:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994220029400. 02000000:00000001:0.0:1775768289.971263:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768289.971264:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768289.971264:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:0.0:1775768289.971265:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768289.971266:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543f800. 02000000:00000001:0.0:1775768289.971267:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768289.971267:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768289.971268:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207889880. 00000100:00000001:0.0:1775768289.971269:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768289.971270:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768289.971270:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768289.971272:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207888700 x1862022294298112/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768314 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768289.971275:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775768289.971276:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775768289.971277:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99422002a800. 02000000:00000001:0.0:1775768289.971278:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775768289.971279:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775768289.971279:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:0.0:1775768289.971281:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775768289.971281:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543fc00. 02000000:00000001:0.0:1775768289.971282:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775768289.971283:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775768289.971283:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207888700. 00000100:00000001:0.0:1775768289.971284:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775768289.971285:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768289.971286:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768289.971288:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768289.971289:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768289.971290:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775768289.971291:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768290.033235:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768290.033241:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768290.033247:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768290.033249:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768290.033252:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768290.033253:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768290.033256:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768290.033258:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768290.033261:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768290.033262:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768290.033279:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768290.033281:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768290.033284:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768290.097220:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768290.097224:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768290.097226:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768290.097230:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768290.097233:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768290.097236:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768290.097238:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768290.097240:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768290.097241:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768290.993250:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768290.993253:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768290.993261:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768290.993261:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768290.993264:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768290.993264:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768290.993268:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:0.0:1775768290.993271:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775768290.993271:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775768290.993277:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768290.993278:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768290.993279:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768290.993280:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768290.993284:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768290.993284:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775768290.993287:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:0.0:1775768290.993289:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768290.993296:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768290.993297:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768290.993302:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768290.993303:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768290.993304:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768290.993311:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768290.993315:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768290.993317:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768290.993323:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768290.993326:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768291.057249:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768291.057253:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768291.057257:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768291.057258:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768291.057261:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768291.057262:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768291.057264:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768291.057265:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768291.057267:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768291.057268:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768291.057313:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768291.057315:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768291.057318:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768291.121233:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768291.121238:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768291.121240:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768291.121244:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768291.121246:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768291.121249:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768291.121251:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768291.121253:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768291.121255:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768291.313264:0:15036:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4300216125, at 4300216128 00000100:00000010:0.0:1775768291.313280:0:15036:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3d80. 00000100:00000040:1.0:1775768291.313320:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4300216125, at 4300216128 00000100:00000010:1.0:1775768291.313335:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942078a3720. 00000100:00000001:3.0:1775768292.017287:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768292.017288:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768292.017296:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768292.017297:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768292.017298:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768292.017302:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768292.017308:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:2.0:1775768292.017308:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775768292.017309:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768292.017316:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768292.017317:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1775768292.017319:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:0.0:1775768292.017323:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768292.017323:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768292.017324:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768292.017330:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768292.017332:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768292.017332:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768292.017337:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768292.017338:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768292.017347:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768292.017348:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768292.017352:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768292.017356:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768292.017357:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768292.017361:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768292.017364:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768292.081290:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768292.081298:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768292.081300:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768292.081303:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768292.081308:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768292.081310:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768292.081312:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768292.081313:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768292.081316:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768292.081318:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768292.081321:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768292.081321:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768292.081324:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768292.081325:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768292.081328:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768292.081329:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768292.081350:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768292.081352:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768292.081354:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768292.145313:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768292.145323:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768292.145326:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768292.145334:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768292.145338:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768292.145343:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768292.145345:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768292.145347:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768292.145350:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768293.041312:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768293.041312:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768293.041321:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768293.041322:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768293.041324:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768293.041325:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768293.041332:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775768293.041333:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768293.041334:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768293.041341:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768293.041341:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768293.041342:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768293.041343:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768293.041347:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768293.041347:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768293.041348:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768293.041353:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768293.041353:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768293.041356:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768293.041358:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768293.041361:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768293.041361:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768293.041365:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768293.041369:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768293.041369:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768293.041372:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768293.041376:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768293.105306:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768293.105315:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768293.105322:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768293.105323:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768293.105328:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768293.105329:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768293.105333:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768293.105334:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768293.105338:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768293.105339:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768293.105391:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768293.105395:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768293.105398:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768293.169309:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768293.169321:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768293.169322:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768293.169327:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768293.169330:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768293.169334:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768293.169337:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768293.169339:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768293.169342:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768294.065274:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768294.065274:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768294.065279:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768294.065281:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768294.065282:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768294.065286:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768294.065287:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768294.065289:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768294.065296:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768294.065297:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:1.0:1775768294.065297:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768294.065309:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768294.065309:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1775768294.065310:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768294.065315:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768294.065316:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768294.065317:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768294.065324:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768294.065327:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768294.065327:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768294.065334:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768294.065335:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768294.065338:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768294.065341:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768294.065342:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768294.065346:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768294.065351:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768294.129261:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768294.129269:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768294.129275:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768294.129276:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768294.129280:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768294.129282:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768294.129286:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768294.129287:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768294.129299:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768294.129303:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768294.129324:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768294.129327:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768294.129330:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768294.193348:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768294.193357:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768294.193359:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768294.193366:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768294.193370:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768294.193379:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768294.193382:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768294.193384:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768294.193386:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768295.089300:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768295.089300:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768295.089302:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768295.089311:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768295.089311:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768295.089312:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768295.089314:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768295.089315:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768295.089317:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768295.089323:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:1.0:1775768295.089323:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775768295.089325:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768295.089329:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768295.089330:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768295.089332:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768295.089334:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768295.089335:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768295.089337:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768295.089339:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768295.089343:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768295.089346:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768295.089347:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768295.089350:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768295.089353:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768295.089354:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768295.089358:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768295.089361:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768295.089430:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768295.089436:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775768295.089440:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768295.089444:0:8:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768295.089448:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768295.089458:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5547 00000400:00000200:0.0:1775768295.089469:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768295.089476:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768295.089480:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768295.089484:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768295.089488:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768295.089490:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768295.089493:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768295.089497:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768295.089499:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:0.0:1775768295.089502:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768295.089505:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:0.0:1775768295.089508:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:0.0:1775768295.089511:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:0.0:1775768295.089514:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768295.089517:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768295.089519:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768295.089521:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768295.089524:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:0.0:1775768295.089528:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768295.089530:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768295.089533:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768295.089537:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768295.089540:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768295.089542:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775768295.089545:0:8:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768295.089561:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fc380. 00000020:00000040:0.0:1775768295.089565:0:8:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:0.0:1775768295.089570:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768295.089572:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768295.089575:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768295.089579:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768295.089584:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768295.089593:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002a800. 00000100:00000001:0.0:1775768295.089600:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768295.089610:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fc380 x1862022294298240/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768295.089620:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768295.089626:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768295.089630:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002957419c] to pc [ptlrpcd_rcv+-1] req@ffff9942099fc380 x1862022294298240/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768295.089651:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768295.089654:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768295.089658:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775768295.089659:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768295.089661:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768295.089665:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768295.089668:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5547 00000400:00000200:0.0:1775768295.089672:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768295.089676:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:2.0:1775768295.089677:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:0.0:1775768295.089678:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1775768295.089683:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1775768295.089684:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768295.089687:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768295.089689:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768295.089690:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1775768295.089692:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768295.089695:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768295.089697:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000100:00000040:2.0:1775768295.089699:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fc380 x1862022294298240/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775768295.089700:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768295.089703:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:0.0:1775768295.089707:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:0.0:1775768295.089711:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000001:2.0:1775768295.089715:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000040:0.0:1775768295.089718:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768295.089718:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768295.089722:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1775768295.089723:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:0.0:1775768295.089724:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768295.089727:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 02000000:00000001:2.0:1775768295.089727:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:0.0:1775768295.089733:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 02000000:00000001:2.0:1775768295.089733:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768295.089738:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00100000:2.0:1775768295.089741:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294298240:192.168.202.156@tcp:250:kworker.0 00000100:00000001:0.0:1775768295.089742:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768295.089745:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1775768295.089749:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768295.089753:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768295.089753:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768295.089757:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768295.089757:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768295.089760:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:2.0:1775768295.089762:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:0.0:1775768295.089766:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 02000000:00000010:2.0:1775768295.089768:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f000. 00000008:00000001:0.0:1775768295.089770:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768295.089772:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768295.089773:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768295.089779:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fea00. 00000400:00000200:2.0:1775768295.089780:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000040:0.0:1775768295.089783:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:0.0:1775768295.089787:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768295.089788:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768295.089790:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768295.089791:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768295.089793:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775768295.089794:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 02000000:00000010:0.0:1775768295.089797:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220029400. 00000100:00000001:0.0:1775768295.089803:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775768295.089807:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00100000:0.0:1775768295.089808:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fea00 x1862022294298368/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:2.0:1775768295.089815:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294298240, portal 25 00000100:00000001:0.0:1775768295.089816:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768295.089820:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768295.089821:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768295.089824:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641197440 : -112966068354176 : ffff9942099fc380) 00000100:00000040:0.0:1775768295.089825:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008d4eaacc] to pc [ptlrpcd_rcv+-1] req@ffff9942099fea00 x1862022294298368/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775768295.089833:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fc380 x1862022294298240/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768320 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768295.089838:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768295.089842:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775768295.089847:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1775768295.089848:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775768295.089851:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768295.089853:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:2.0:1775768295.089854:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000100:00000001:0.0:1775768295.089859:0:8:0:(import.c:484:import_select_connection()) Process entered 00000400:00000010:2.0:1775768295.089859:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00080000:0.0:1775768295.089864:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5547 00000100:00000200:2.0:1775768295.089864:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294298240, offset 0 00000400:00000200:0.0:1775768295.089869:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768295.089872:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768295.089874:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768295.089879:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000200:2.0:1775768295.089882:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00000001:0.0:1775768295.089883:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768295.089888:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768295.089890:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768295.089892:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768295.089895:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000400:00000200:2.0:1775768295.089895:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:0.0:1775768295.089899:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768295.089901:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:0.0:1775768295.089904:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000400:00000200:2.0:1775768295.089906:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000020:00000001:0.0:1775768295.089907:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:0.0:1775768295.089913:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000200:2.0:1775768295.089914:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1775768295.089917:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768295.089921:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000200:2.0:1775768295.089921:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000100:00000001:0.0:1775768295.089925:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000200:2.0:1775768295.089927:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000020:00000040:0.0:1775768295.089930:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000400:00000200:2.0:1775768295.089931:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00080000:0.0:1775768295.089935:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768295.089938:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768295.089940:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:2.0:1775768295.089944:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225503:225503:256:4294967295] 192.168.202.156@tcp LPNI seq info [225499:225499:8:4294967295] 02000000:00000001:0.0:1775768295.089950:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1775768295.089950:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:0.0:1775768295.089955:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768295.089957:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775768295.089959:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775768295.089963:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768295.089967:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768295.089969:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000100:00000010:0.0:1775768295.089972:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fdc00. 00000020:00000040:0.0:1775768295.089976:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000800:00000200:2.0:1775768295.089979:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 02000000:00000001:0.0:1775768295.089980:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768295.089983:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768295.089985:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768295.089986:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000800:00000010:2.0:1775768295.089987:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 02000000:00000001:0.0:1775768295.089988:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768295.089994:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220029800. 00000800:00000200:2.0:1775768295.089997:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:0.0:1775768295.089998:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1775768295.090000:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000100:00100000:0.0:1775768295.090006:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fdc00 x1862022294298496/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:2.1:1775768295.090008:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768295.090013:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000100:00000001:0.0:1775768295.090021:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000100:2.0:1775768295.090021:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768295.090024:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000100:00000001:0.0:1775768295.090025:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768295.090031:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:0.0:1775768295.090032:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001372d9ae] to pc [ptlrpcd_rcv+-1] req@ffff9942099fdc00 x1862022294298496/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1775768295.090038:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000100:00000001:0.0:1775768295.090055:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768295.090059:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:2.0:1775768295.090060:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768295.090063:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00080000:0.0:1775768295.090064:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775768295.090067:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768295.090068:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775768295.090068:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1775768295.090073:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00000200:2.0:1775768295.090073:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fc380 x1862022294298240/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768320 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768295.090079:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5547 00000400:00000200:0.0:1775768295.090085:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768295.090086:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000400:00000200:0.0:1775768295.090092:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000040:2.0:1775768295.090092:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fc380 x1862022294298240/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768320 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768295.090097:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1775768295.090100:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768295.090104:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768295.090105:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775768295.090108:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768295.090111:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768295.090113:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:2.0:1775768295.090113:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768295.090118:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768295.090120:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:0.0:1775768295.090123:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:2.0:1775768295.090123:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000040:0.0:1775768295.090126:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000100:00000001:2.0:1775768295.090128:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4300221245 00000020:00000001:0.0:1775768295.090130:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:0.0:1775768295.090134:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000010:2.0:1775768295.090136:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d6540. 00000100:00000040:0.0:1775768295.090138:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768295.090142:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768295.090144:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768295.090145:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775768295.090147:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768295.090150:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00000001:2.0:1775768295.090152:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4300221245 00000100:00080000:0.0:1775768295.090174:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768295.090176:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768295.090178:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768295.090181:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768295.090183:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775768295.090183:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d6cc0. 00000100:00000001:0.0:1775768295.090186:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775768295.090189:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000100:00000001:2.0:1775768295.090190:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000008:00000001:0.0:1775768295.090192:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768295.090194:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768295.090196:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000010:0.0:1775768295.090198:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fce00. 00000100:00000001:2.0:1775768295.090199:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1775768295.090200:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:0.0:1775768295.090204:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768295.090205:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768295.090205:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1775768295.090208:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768295.090209:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768295.090211:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768295.090213:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768295/real 1775768295] req@ffff9942099fc380 x1862022294298240/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1775768295.090215:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002bc00. 00000100:00000001:0.0:1775768295.090219:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768295.090225:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fce00 x1862022294298624/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:2.0:1775768295.090231:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1775768295.090232:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768295.090234:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1775768295.090234:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768295.090237:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000100:00000040:0.0:1775768295.090238:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000060e5e9d1] to pc [ptlrpcd_rcv+-1] req@ffff9942099fce00 x1862022294298624/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1775768295.090243:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000100:00000001:0.0:1775768295.090245:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775768295.090246:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00080000:0.0:1775768295.090248:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775768295.090250:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00080000:0.0:1775768295.090251:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775768295.090253:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768295.090254:0:8:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:2.0:1775768295.090256:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fc380 x1862022294298240/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768295.090257:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768295.090261:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5547 00000400:00000200:0.0:1775768295.090268:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768295.090272:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000200:2.0:1775768295.090275:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fc380 x1862022294298240/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768295.090282:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00080000:0.0:1775768295.090283:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1775768295.090286:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768295.090289:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:0.0:1775768295.090290:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768295.090292:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768295.090293:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768295.090296:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768295.090298:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768295.090301:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000100:00100000:2.0:1775768295.090301:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fc380 x1862022294298240/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1775768295.090303:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:0.0:1775768295.090305:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768295.090307:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:0.0:1775768295.090311:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:0.0:1775768295.090314:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000001:2.0:1775768295.090317:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768295.090318:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768295.090322:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768295.090324:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768295.090325:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000040:2.0:1775768295.090327:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fc380 x1862022294298240/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768320 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1775768295.090328:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00080000:0.0:1775768295.090332:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768295.090335:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768295.090338:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768295.090341:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768295.090344:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768295.090346:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775768295.090346:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00100000:0.0:1775768295.090347:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:0.0:1775768295.090352:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768295.090353:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768295.090353:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:0.0:1775768295.090358:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fca80. 00000100:00000001:2.0:1775768295.090358:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768295.090359:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:0.0:1775768295.090360:0:8:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:0.0:1775768295.090364:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1775768295.090364:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 02000000:00000001:0.0:1775768295.090365:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768295.090368:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768295.090369:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1775768295.090369:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:0.0:1775768295.090372:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768295.090375:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220028400. 00000100:00080000:2.0:1775768295.090375:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:0.0:1775768295.090381:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768295.090384:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00100000:0.0:1775768295.090385:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fca80 x1862022294298752/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775768295.090388:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:0.0:1775768295.090395:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768295.090397:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768295.090397:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fc380 x1862022294298240/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768320 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775768295.090403:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000046a3ca2f] to pc [ptlrpcd_rcv+-1] req@ffff9942099fca80 x1862022294298752/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768295.090417:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768295.090420:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294298240:192.168.202.156@tcp:250:kworker.0 00000100:00000040:0.0:1775768295.090422:0:8:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5557) 00000100:00000001:2.0:1775768295.090429:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768295.090431:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768295.090435:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fc380 x1862022294298240/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768320 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768295.090443:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768295.090445:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768295.090448:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f000. 02000000:00000001:2.0:1775768295.090451:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768295.090453:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768295.090455:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:2.0:1775768295.090457:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768295.090459:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002a800. 02000000:00000001:2.0:1775768295.090462:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768295.090464:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768295.090466:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fc380. 00000100:00000001:2.0:1775768295.090469:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768295.090470:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768295.090472:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768295.090479:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768295.090480:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768295.090483:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768295.090484:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768295.090486:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768295.090489:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fea00 x1862022294298368/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768295.090512:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768295.090514:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768295.090516:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768295.090517:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768295.090519:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768295.090522:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294298368:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768295.090525:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768295.090527:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768295.090529:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768295.090530:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768295.090533:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f000. 02000000:00000001:2.0:1775768295.090535:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768295.090539:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768295.090541:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768295.090545:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768295.090547:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294298368, portal 10 00000100:00000001:2.0:1775768295.090549:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768295.090550:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641207296 : -112966068344320 : ffff9942099fea00) 00000100:00000040:2.0:1775768295.090556:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fea00 x1862022294298368/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768320 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768295.090563:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768295.090565:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768295.090567:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768295.090569:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294298368, offset 0 00000400:00000200:2.0:1775768295.090573:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768295.090576:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768295.090580:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768295.090582:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768295.090585:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768295.090587:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768295.090589:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768295.090592:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225504:225504:256:4294967295] 192.168.202.156@tcp LPNI seq info [225500:225500:8:4294967295] 00000400:00000200:2.0:1775768295.090596:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768295.090602:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768295.090605:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768295.090608:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768295.090611:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768295.090614:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775768295.090618:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768295.090620:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1775768295.090624:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768295.090626:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768295.090628:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768295.090632:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768295.090634:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768295.090636:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768295.090638:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768295.090641:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fea00 x1862022294298368/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768320 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768295.090649:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768295.090651:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fea00 x1862022294298368/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768320 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768295.090657:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768295.090659:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768295.090661:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768295.090665:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:2.0:1775768295.090668:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768295.090669:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300219923 00000100:00000001:2.0:1775768295.090672:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300221245 00000100:00000001:2.0:1775768295.090674:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300219923 00000100:00000001:2.0:1775768295.090677:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300221245 00000100:00000001:2.0:1775768295.090679:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768295.090681:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768295.090683:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768295.090686:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768295/real 1775768295] req@ffff9942099fea00 x1862022294298368/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768295.090693:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768295.090694:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768295.090696:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768295.090698:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768295.090699:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768295.090701:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768295.090704:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fea00 x1862022294298368/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768295.090711:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fea00 x1862022294298368/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768295.090717:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768295.090718:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768295.090720:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768295.090721:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768295.090724:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fea00 x1862022294298368/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768295.090730:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768295.090734:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fea00 x1862022294298368/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768320 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768295.090740:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768295.090742:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768295.090744:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768295.090745:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768295.090747:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768295.090750:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768295.090752:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768295.090756:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768295.090758:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768295.090762:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fea00 x1862022294298368/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768320 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768295.090769:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294298368:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768295.090772:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768295.090775:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fdc00 x1862022294298496/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768295.090781:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768295.090782:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768295.090784:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768295.090786:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768295.090788:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768295.090790:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294298496:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768295.090793:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768295.090795:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768295.090797:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768295.090798:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768295.090801:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1ec00. 02000000:00000001:2.0:1775768295.090803:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768295.090806:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768295.090808:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768295.090811:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768295.090813:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294298496, portal 10 00000100:00000001:2.0:1775768295.090815:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768295.090817:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641203712 : -112966068347904 : ffff9942099fdc00) 00000100:00000040:2.0:1775768295.090821:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fdc00 x1862022294298496/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768320 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768295.090828:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768295.090831:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768295.090833:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768295.090835:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294298496, offset 0 00000400:00000200:2.0:1775768295.090839:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768295.090842:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768295.090845:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768295.090847:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768295.090850:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768295.090852:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768295.090854:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768295.090857:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225505:225505:256:4294967295] 192.168.202.156@tcp LPNI seq info [225501:225501:8:4294967295] 00000400:00000200:2.0:1775768295.090861:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768295.090866:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768295.090870:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768295.090873:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768295.090877:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768295.090879:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775768295.090883:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768295.090885:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1775768295.090889:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768295.090891:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768295.090893:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768295.090897:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768295.090899:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768295.090901:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768295.090903:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768295.090906:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fdc00 x1862022294298496/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768320 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768295.090913:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768295.090916:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fdc00 x1862022294298496/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768320 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768295.090922:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768295.090924:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768295.090926:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768295.090929:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:2.0:1775768295.090932:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768295.090934:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300219915 00000100:00000001:2.0:1775768295.090936:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300221245 00000100:00000001:2.0:1775768295.090938:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300219915 00000100:00000001:2.0:1775768295.090940:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300221245 00000100:00000001:2.0:1775768295.090942:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768295.090944:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768295.090946:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768295.090949:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768295/real 1775768295] req@ffff9942099fdc00 x1862022294298496/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768295.090956:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768295.090958:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768295.090959:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768295.090962:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768295.090963:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768295.090966:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768295.090968:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fdc00 x1862022294298496/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768295.090976:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fdc00 x1862022294298496/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768295.090983:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768295.090984:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768295.090986:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768295.090987:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768295.090991:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fdc00 x1862022294298496/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768295.090997:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768295.091001:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fdc00 x1862022294298496/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768320 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768295.091008:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768295.091010:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768295.091012:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768295.091013:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768295.091015:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768295.091018:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768295.091020:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768295.091023:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768295.091025:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768295.091029:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fdc00 x1862022294298496/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768320 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768295.091036:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294298496:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768295.091051:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768295.091054:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fce00 x1862022294298624/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768295.091061:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768295.091062:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768295.091064:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768295.091066:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768295.091068:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768295.091070:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294298624:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768295.091073:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768295.091075:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768295.091076:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768295.091078:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768295.091081:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1fc00. 02000000:00000001:2.0:1775768295.091083:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768295.091087:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768295.091089:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768295.091092:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768295.091095:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294298624, portal 4 00000100:00000001:2.0:1775768295.091097:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768295.091098:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641200128 : -112966068351488 : ffff9942099fce00) 00000100:00000040:2.0:1775768295.091103:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fce00 x1862022294298624/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768320 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768295.091109:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768295.091111:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768295.091113:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768295.091115:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294298624, offset 0 00000400:00000200:2.0:1775768295.091119:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768295.091122:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768295.091125:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768295.091128:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768295.091130:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768295.091132:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768295.091135:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768295.091137:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225506:225506:256:4294967295] 192.168.202.156@tcp LPNI seq info [225502:225502:8:4294967295] 00000400:00000200:2.0:1775768295.091141:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768295.091147:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768295.091151:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768295.091154:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768295.091173:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768295.091175:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775768295.091179:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768295.091181:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1775768295.091185:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768295.091186:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768295.091189:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768295.091192:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768295.091195:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768295.091197:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768295.091199:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768295.091202:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fce00 x1862022294298624/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768320 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768295.091209:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768295.091212:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fce00 x1862022294298624/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768320 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768295.091218:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768295.091220:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768295.091222:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768295.091225:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:2.0:1775768295.091228:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768295.091230:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300219939 00000100:00000001:2.0:1775768295.091232:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300221245 00000100:00000001:2.0:1775768295.091234:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300219939 00000100:00000001:2.0:1775768295.091236:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300221245 00000100:00000001:2.0:1775768295.091238:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768295.091240:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768295.091242:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768295.091245:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768295/real 1775768295] req@ffff9942099fce00 x1862022294298624/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768295.091251:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768295.091253:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768295.091254:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768295.091257:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768295.091258:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768295.091260:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768295.091263:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fce00 x1862022294298624/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768295.091270:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fce00 x1862022294298624/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768295.091276:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768295.091278:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768295.091279:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768295.091281:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768295.091284:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fce00 x1862022294298624/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768295.091290:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768295.091294:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fce00 x1862022294298624/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768320 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768295.091300:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768295.091302:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768295.091304:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768295.091305:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768295.091307:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768295.091311:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768295.091313:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768295.091316:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768295.091318:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768295.091321:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fce00 x1862022294298624/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768320 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768295.091329:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294298624:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768295.091332:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768295.091335:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fca80 x1862022294298752/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768295.091341:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768295.091342:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768295.091344:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768295.091346:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768295.091348:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768295.091350:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294298752:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768295.091353:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768295.091355:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768295.091356:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768295.091357:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768295.091360:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1d000. 02000000:00000001:2.0:1775768295.091362:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768295.091365:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768295.091367:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768295.091370:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768295.091372:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294298752, portal 4 00000100:00000001:2.0:1775768295.091374:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768295.091375:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641199232 : -112966068352384 : ffff9942099fca80) 00000100:00000040:2.0:1775768295.091380:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fca80 x1862022294298752/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768320 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768295.091386:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768295.091388:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768295.091390:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768295.091392:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294298752, offset 0 00000400:00000200:2.0:1775768295.091395:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768295.091399:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768295.091402:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768295.091404:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768295.091407:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768295.091409:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768295.091411:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768295.091414:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225507:225507:256:4294967295] 192.168.202.156@tcp LPNI seq info [225503:225503:8:4294967295] 00000400:00000200:2.0:1775768295.091418:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768295.091424:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768295.091428:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768295.091431:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768295.091434:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768295.091437:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775768295.091441:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768295.091443:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1775768295.091446:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768295.091448:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768295.091451:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768295.091454:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768295.091456:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768295.091458:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768295.091460:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768295.091463:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fca80 x1862022294298752/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768320 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768295.091470:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768295.091472:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fca80 x1862022294298752/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768320 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768295.091478:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768295.091480:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768295.091482:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768295.091485:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:2.0:1775768295.091487:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768295.091490:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300219942 00000100:00000001:2.0:1775768295.091492:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300221245 00000100:00000001:2.0:1775768295.091494:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300219942 00000100:00000001:2.0:1775768295.091496:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300221245 00000100:00000001:2.0:1775768295.091498:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768295.091500:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768295.091502:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768295.091505:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768295/real 1775768295] req@ffff9942099fca80 x1862022294298752/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768295.091513:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768295.091514:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768295.091515:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768295.091518:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768295.091519:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768295.091521:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768295.091524:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fca80 x1862022294298752/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768295.091532:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fca80 x1862022294298752/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768295.091538:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768295.091539:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768295.091540:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768295.091542:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768295.091545:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fca80 x1862022294298752/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768295.091551:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768295.091554:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fca80 x1862022294298752/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768320 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768295.091561:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768295.091563:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768295.091564:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768295.091566:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768295.091567:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768295.091571:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768295.091574:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768295.091576:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768295.091579:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768295.091583:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fca80 x1862022294298752/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768320 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768295.091590:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294298752:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768295.091593:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768295.091595:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768295.091598:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fea00 x1862022294298368/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768320 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768295.091603:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768295.091605:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768295.091607:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f000. 02000000:00000001:2.0:1775768295.091609:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768295.091610:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768295.091612:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:2.0:1775768295.091614:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768295.091615:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029400. 02000000:00000001:2.0:1775768295.091618:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768295.091619:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768295.091620:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fea00. 00000100:00000001:2.0:1775768295.091622:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768295.091623:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768295.091625:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768295.091628:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fdc00 x1862022294298496/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768320 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768295.091633:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768295.091635:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768295.091636:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1ec00. 02000000:00000001:2.0:1775768295.091639:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768295.091640:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768295.091641:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:2.0:1775768295.091643:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768295.091644:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029800. 02000000:00000001:2.0:1775768295.091646:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768295.091647:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768295.091648:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fdc00. 00000100:00000001:2.0:1775768295.091650:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768295.091652:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768295.091653:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768295.091656:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fce00 x1862022294298624/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768320 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768295.091661:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768295.091662:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768295.091664:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1fc00. 02000000:00000001:2.0:1775768295.091665:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768295.091666:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768295.091667:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:2.0:1775768295.091669:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768295.091670:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002bc00. 02000000:00000001:2.0:1775768295.091672:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768295.091673:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768295.091674:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fce00. 00000100:00000001:2.0:1775768295.091677:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768295.091678:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768295.091679:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768295.091682:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fca80 x1862022294298752/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768320 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768295.091688:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768295.091689:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768295.091690:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1d000. 02000000:00000001:2.0:1775768295.091693:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768295.091694:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768295.091695:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:2.0:1775768295.091697:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768295.091698:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028400. 02000000:00000001:2.0:1775768295.091700:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768295.091701:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768295.091702:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fca80. 00000100:00000001:2.0:1775768295.091704:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768295.091705:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768295.091707:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768295.091710:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768295.091712:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768295.091714:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768295.091715:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768295.153292:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768295.153299:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768295.153312:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768295.153313:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768295.153316:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768295.153317:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768295.153319:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768295.153320:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768295.153322:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768295.153323:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768295.153385:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768295.153390:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768295.153393:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768295.217282:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768295.217290:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768295.217292:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768295.217299:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768295.217304:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768295.217309:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768295.217312:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768295.217315:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768295.217319:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768296.113218:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768296.113223:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768296.113225:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768296.113230:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768296.113233:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768296.113235:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768296.113237:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768296.113240:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768296.113244:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768296.113248:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768296.113252:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768296.113255:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768296.113260:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768296.113263:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768296.113264:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768296.113266:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768296.113271:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768296.113271:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768296.113275:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768296.113276:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768296.113282:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768296.113283:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775768296.113290:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768296.113293:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768296.113295:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768296.113297:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768296.113299:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768296.177279:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768296.177284:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768296.177289:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768296.177290:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768296.177293:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768296.177294:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768296.177297:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768296.177298:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768296.177301:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768296.177302:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768296.177319:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768296.177328:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768296.177332:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768296.241287:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768296.241294:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768296.241296:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768296.241301:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768296.241304:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768296.241307:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768296.241309:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768296.241311:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768296.241314:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768296.433275:0:15036:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4300221245, at 4300221248 00000100:00000040:1.0:1775768296.433278:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4300221245, at 4300221248 00000100:00000010:0.0:1775768296.433295:0:15036:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d6540. 00000100:00000010:1.0:1775768296.433299:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d6cc0. 00000100:00000001:0.0:1775768297.137252:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768297.137253:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768297.137260:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768297.137261:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768297.137263:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768297.137265:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768297.137268:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:1.0:1775768297.137272:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768297.137273:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768297.137277:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768297.137278:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768297.137281:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768297.137283:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768297.137284:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768297.137294:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768297.137295:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768297.137298:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768297.137303:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768297.137396:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768297.137399:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768297.137400:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768297.137403:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768297.137405:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768297.137406:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768297.137408:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768297.137409:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768297.137410:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768297.201300:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768297.201306:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768297.201311:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768297.201312:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768297.201316:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768297.201317:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768297.201326:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768297.201327:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768297.201331:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768297.201332:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768297.201390:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768297.201395:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768297.201398:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768297.265280:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768297.265288:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768297.265291:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768297.265298:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768297.265302:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768297.265307:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768297.265310:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768297.265313:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768297.265316:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768298.161230:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768298.161237:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768298.161239:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768298.161245:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768298.161250:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768298.161256:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768298.161259:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768298.161263:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768298.161267:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768298.161285:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768298.161293:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768298.161296:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768298.161301:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775768298.161308:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768298.161308:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768298.161313:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768298.161314:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768298.161318:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768298.161319:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768298.161323:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768298.161325:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768298.161328:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768298.161330:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768298.161334:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768298.161337:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768298.161340:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768298.161344:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768298.225279:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768298.225285:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768298.225300:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768298.225304:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768298.225306:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768298.225309:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768298.225312:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768298.225314:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768298.225316:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768298.225317:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768298.225321:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768298.225322:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768298.225323:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768298.225324:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768298.225325:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768298.225326:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768298.225327:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768298.225329:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768298.225333:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768298.225335:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768298.225336:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768298.225337:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768298.225339:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768298.225340:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768298.225341:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768298.225343:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768298.225346:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768298.225348:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768298.225349:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768298.225350:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768298.225352:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768298.225353:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768298.225354:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768298.225356:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768298.225380:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768298.225384:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768298.225387:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768298.289287:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768298.289294:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768298.289296:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768298.289302:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768298.289305:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768298.289310:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768298.289312:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768298.289314:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768298.289315:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768299.185267:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768299.185272:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768299.185274:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768299.185278:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768299.185279:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768299.185282:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768299.185282:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768299.185285:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:0.0:1775768299.185289:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:3.0:1775768299.185290:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775768299.185291:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768299.185293:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768299.185296:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768299.185298:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:0.0:1775768299.185300:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768299.185302:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768299.185302:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768299.185309:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768299.185310:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768299.185313:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768299.185315:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768299.185317:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768299.185320:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768299.185323:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768299.185327:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768299.185330:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768299.185334:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768299.249279:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768299.249287:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768299.249293:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768299.249294:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768299.249298:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768299.249299:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768299.249303:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768299.249304:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768299.249307:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768299.249308:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768299.249379:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768299.249383:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768299.249387:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768299.313306:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768299.313314:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768299.313317:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768299.313324:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768299.313329:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768299.313333:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768299.313336:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768299.313339:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768299.313342:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768300.209304:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768300.209304:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768300.209309:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768300.209311:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768300.209312:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768300.209313:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768300.209316:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768300.209316:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768300.209319:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768300.209324:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:1.0:1775768300.209324:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775768300.209324:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768300.209329:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768300.209330:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768300.209331:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768300.209333:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768300.209334:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768300.209335:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768300.209336:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768300.209340:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768300.209342:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768300.209343:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768300.209346:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768300.209349:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768300.209350:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768300.209353:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768300.209357:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768300.209375:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768300.209382:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775768300.209386:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775768300.209390:0:8:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775768300.209395:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775768300.209404:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5552 00000400:00000200:2.0:1775768300.209415:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768300.209422:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775768300.209426:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775768300.209430:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768300.209434:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768300.209437:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768300.209441:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775768300.209445:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775768300.209448:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:2.0:1775768300.209452:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000080:00000001:3.0:1775768300.209453:0:36:0:(rw.c:1481:ll_writepages()) Process entered 00000080:00000001:3.0:1775768300.209459:0:36:0:(file.c:5250:cl_sync_file_range()) Process entered 00000020:00000001:3.0:1775768300.209463:0:36:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000040:2.0:1775768300.209466:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:2.0:1775768300.209470:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000010:3.0:1775768300.209470:0:36:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff99420693e000. 00000020:00000001:2.0:1775768300.209475:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:2.0:1775768300.209479:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000010:3.0:1775768300.209479:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcf800. 00000100:00000001:2.0:1775768300.209483:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768300.209486:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000010:3.0:1775768300.209487:0:36:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210dce800. 00000100:00000001:2.0:1775768300.209489:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:2.0:1775768300.209493:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000020:00000010:3.0:1775768300.209494:0:36:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207981900. 00000100:00080000:2.0:1775768300.209498:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775768300.209501:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:3.0:1775768300.209504:0:36:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994208b50000. 02000000:00000001:2.0:1775768300.209505:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775768300.209509:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000010:3.0:1775768300.209512:0:36:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994218000300. 02000000:00000001:2.0:1775768300.209513:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768300.209517:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775768300.209520:0:8:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:3.0:1775768300.209521:0:36:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994207e08e20. 00000100:00000010:2.0:1775768300.209526:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dff480. 00000080:00000010:3.0:1775768300.209529:0:36:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99420541b1f8. 00000020:00000040:2.0:1775768300.209531:0:8:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 00000080:00000010:3.0:1775768300.209535:0:36:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077c0248. 02000000:00000001:2.0:1775768300.209537:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775768300.209540:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1775768300.209541:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcf000. 00000100:00000001:2.0:1775768300.209545:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000008:00000010:3.0:1775768300.209548:0:36:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942086f8420. 02000000:00000001:2.0:1775768300.209550:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00020000:00000010:3.0:1775768300.209553:0:36:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994207e8e400. 02000000:00000001:2.0:1775768300.209556:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:3.0:1775768300.209558:0:36:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994209270a90. 00000080:00000010:3.0:1775768300.209563:0:36:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942090e33c0. 02000000:00000010:2.0:1775768300.209565:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1d000. 00000020:00000001:3.0:1775768300.209567:0:36:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107590086664 : -112966119464952 : ffff99420693e008) 00000020:00001000:3.0:1775768300.209571:0:36:0:(cl_object.c:831:cl_env_get()) 1@ffff99420693e000 00000100:00000001:2.0:1775768300.209574:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768300.209576:0:36:0:(cl_object.c:151:cl_object_top()) ffff99420791a5f8 -> ffff99420791a5f8 00000020:00000001:3.0:1775768300.209580:0:36:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:3.0:1775768300.209584:0:36:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000100:00100000:2.0:1775768300.209586:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dff480 x1862022294298880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000080:00200000:3.0:1775768300.209587:0:36:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x10176:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:3.0:1775768300.209594:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:3.0:1775768300.209597:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:3.0:1775768300.209600:0:36:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768300.209604:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00020000:00000002:3.0:1775768300.209605:0:36:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x10176:0x0]io ffff9942077c02a8 type 6 ignore/verify layout 1/0 00000100:00000001:2.0:1775768300.209609:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:3.0:1775768300.209611:0:36:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207dfb6c8) owner 0000000000000000/ffff994201348000 00020000:00000001:3.0:1775768300.209617:0:36:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00000100:00000040:2.0:1775768300.209618:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093a15639] to pc [ptlrpcd_rcv+-1] req@ffff994207dff480 x1862022294298880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:3.0:1775768300.209620:0:36:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:3.0:1775768300.209624:0:36:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00020000:00000001:3.0:1775768300.209627:0:36:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768300.209632:0:36:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:3.0:1775768300.209635:0:36:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:3.0:1775768300.209637:0:36:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768300.209639:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:3.0:1775768300.209641:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:3.0:1775768300.209643:0:36:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00000100:00000001:2.0:1775768300.209645:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:3.0:1775768300.209645:0:36:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207dfb6c8) owner 0000000000000000/ffff994201348000 00000020:00000001:3.0:1775768300.209649:0:36:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768300.209651:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000001:3.0:1775768300.209654:0:36:0:(cl_io.c:823:cl_io_loop()) Process entered 00000100:00080000:2.0:1775768300.209656:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000020:00000001:3.0:1775768300.209657:0:36:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000100:00000001:2.0:1775768300.209659:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00020000:00000001:3.0:1775768300.209661:0:36:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00000100:00080000:2.0:1775768300.209662:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768300.209665:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768300.209666:0:8:0:(import.c:484:import_select_connection()) Process entered 00020000:00200000:3.0:1775768300.209666:0:36:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00000001:1.0:1775768300.209670:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1775768300.209671:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5552 00000100:00000001:1.0:1775768300.209672:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00020000:00000002:3.0:1775768300.209672:0:36:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00000400:00000200:2.0:1775768300.209678:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:3.0:1775768300.209679:0:36:0:(lov_io.c:155:lov_sub_get()) Process entered 00000100:00000040:1.0:1775768300.209681:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dff480 x1862022294298880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:3.0:1775768300.209683:0:36:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000400:00000200:2.0:1775768300.209686:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000020:00000001:3.0:1775768300.209687:0:36:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00080000:2.0:1775768300.209691:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000020:00000010:3.0:1775768300.209692:0:36:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff99420693ecf0. 00000100:00000001:1.0:1775768300.209694:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768300.209696:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768300.209697:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000010:3.0:1775768300.209699:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcf600. 02000000:00000001:1.0:1775768300.209700:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1775768300.209701:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1775768300.209703:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1775768300.209705:0:36:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210dce400. 00000100:00000040:2.0:1775768300.209706:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000001:1.0:1775768300.209707:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768300.209711:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000010:3.0:1775768300.209712:0:36:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207981f60. 00000100:00100000:1.0:1775768300.209713:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294298880:192.168.202.156@tcp:250:kworker.0 00000020:00000001:2.0:1775768300.209718:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:1.0:1775768300.209719:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000040:2.0:1775768300.209720:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 02000000:00000001:1.0:1775768300.209723:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:2.0:1775768300.209723:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775768300.209725:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 02000000:00000001:1.0:1775768300.209726:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768300.209730:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 02000000:00000001:1.0:1775768300.209731:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:2.0:1775768300.209734:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 02000000:00000010:1.0:1775768300.209736:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543d800. 02000000:00000001:1.0:1775768300.209739:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768300.209739:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768300.209743:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768300.209745:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000200:1.0:1775768300.209746:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768300.209749:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000010:1.0:1775768300.209751:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c738. 00000020:00000040:2.0:1775768300.209753:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000100:00080000:2.0:1775768300.209757:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000400:00000010:1.0:1775768300.209758:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf65d8. 00000100:00000001:2.0:1775768300.209759:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768300.209763:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000200:1.0:1775768300.209767:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294298880, portal 25 02000000:00000001:2.0:1775768300.209769:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768300.209770:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768300.209773:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611849856 : -112966097701760 : ffff994207dff480) 02000000:00000001:2.0:1775768300.209774:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768300.209777:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000040:1.0:1775768300.209780:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dff480 x1862022294298880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768325 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1775768300.209784:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1775768300.209786:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00000010:3.0:1775768300.209786:0:36:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994208b51650. 00000008:00000001:2.0:1775768300.209790:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768300.209792:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00020000:00000010:3.0:1775768300.209793:0:36:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994218000000. 00000100:00000001:2.0:1775768300.209795:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775768300.209796:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6bb0. 00008000:00000010:3.0:1775768300.209799:0:36:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994207e0ade8. 00000100:00000200:1.0:1775768300.209800:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294298880, offset 0 00000100:00000010:2.0:1775768300.209801:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfce00. 00000020:00000040:2.0:1775768300.209806:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000080:00000010:3.0:1775768300.209808:0:36:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99420541b0a8. 02000000:00000001:2.0:1775768300.209809:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000200:1.0:1775768300.209812:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 02000000:00000001:2.0:1775768300.209812:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:3.0:1775768300.209814:0:36:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077c1da8. 00000100:00000001:2.0:1775768300.209816:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775768300.209819:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000020:00000010:3.0:1775768300.209820:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcec00. 00000400:00000200:1.0:1775768300.209822:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:2.0:1775768300.209822:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768300.209829:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 02000000:00000010:2.0:1775768300.209829:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1fc00. 00000100:00000001:2.0:1775768300.209836:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768300.209837:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768300.209842:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000100:00100000:2.0:1775768300.209845:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfce00 x1862022294299008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775768300.209848:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768300.209852:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768300.209858:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225508:225508:256:4294967295] 192.168.202.156@tcp LPNI seq info [225504:225504:8:4294967295] 00000100:00000001:2.0:1775768300.209862:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1775768300.209864:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768300.209867:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768300.209872:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e5a7debe] to pc [ptlrpcd_rcv+-1] req@ffff994207dfce00 x1862022294299008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775768300.209877:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768300.209887:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00000001:2.0:1775768300.209891:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1775768300.209892:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541300. 00000100:00080000:2.0:1775768300.209894:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:1.0:1775768300.209900:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00080000:2.0:1775768300.209900:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775768300.209903:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775768300.209906:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:1.1:1775768300.209907:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:2.0:1775768300.209913:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775768300.209916:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5552 00000400:00000200:2.0:1775768300.209921:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768300.209926:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768300.209926:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:2.0:1775768300.209931:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1775768300.209932:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768300.209936:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4300226365 00000100:00000040:2.0:1775768300.209937:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000008:00000010:3.0:1775768300.209939:0:36:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942086f8160. 00000100:00000010:1.0:1775768300.209941:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba3c0. 00000100:00000001:2.0:1775768300.209941:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00020000:00000010:3.0:1775768300.209945:0:36:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994207e8ed00. 00000100:00000040:2.0:1775768300.209946:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768300.209949:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:1.0:1775768300.209950:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00008000:00000010:3.0:1775768300.209950:0:36:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994209270ef0. 00000020:00000001:2.0:1775768300.209954:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775768300.209956:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000080:00000010:3.0:1775768300.209957:0:36:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942090e3960. 00000020:00000001:2.0:1775768300.209958:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:1.0:1775768300.209960:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4300226365 00000020:00000040:2.0:1775768300.209961:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:3.0:1775768300.209961:0:36:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107590089976 : -112966119461640 : ffff99420693ecf8) 00000020:00000001:2.0:1775768300.209965:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000010:1.0:1775768300.209967:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff994207cba240. 00000020:00000001:2.0:1775768300.209968:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:2.0:1775768300.209973:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768300.209974:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1775768300.209976:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775768300.209980:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768300.209981:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768300.209983:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:1.0:1775768300.209985:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775768300.209987:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00000001:1.0:1775768300.209989:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768300.210002:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1775768300.210003:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775768300.210007:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768300.210009:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:2.0:1775768300.210010:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1775768300.210013:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 02000000:00000001:2.0:1775768300.210015:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768300.210018:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00001000:3.0:1775768300.210018:0:36:0:(cl_object.c:831:cl_env_get()) 1@ffff99420693ecf0 02000000:00000001:2.0:1775768300.210019:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768300.210021:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768300.210023:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775768300.210024:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1775768300.210025:0:36:0:(cl_object.c:151:cl_object_top()) ffff99420791ae90 -> ffff99420791a5f8 00000008:00100000:2.0:1775768300.210028:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000020:00000001:3.0:1775768300.210028:0:36:0:(cl_io.c:125:__cl_io_init()) Process entered 00000100:00000040:1.0:1775768300.210031:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfce00 x1862022294299008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:2.0:1775768300.210033:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768300.210033:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:3.0:1775768300.210036:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00000001:2.0:1775768300.210037:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768300.210040:0:36:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775768300.210044:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfed80. 00020000:00000001:3.0:1775768300.210046:0:36:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768300.210047:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768300.210049:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775768300.210050:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00020000:00000001:3.0:1775768300.210051:0:36:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107612435560 : -112966097116056 : ffff994207e8e468) 02000000:00000001:1.0:1775768300.210053:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768300.210056:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768300.210057:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000020:00000001:3.0:1775768300.210057:0:36:0:(cl_io.c:399:cl_io_iter_init()) Process entered 02000000:00000001:2.0:1775768300.210059:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768300.210061:0:36:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768300.210063:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775768300.210065:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768300.210065:0:36:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768300.210066:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775768300.210070:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768300.210070:0:36:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768300.210074:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294299008:192.168.202.156@tcp:38:kworker.0 00000020:00000001:3.0:1775768300.210076:0:36:0:(cl_io.c:314:cl_io_lock()) Process entered 02000000:00000010:2.0:1775768300.210078:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1ec00. 00000100:00000001:1.0:1775768300.210079:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00020000:00000001:3.0:1775768300.210079:0:36:0:(lov_io.c:1108:lov_io_lock()) Process entered 02000000:00000001:1.0:1775768300.210084:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1775768300.210084:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768300.210084:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 02000000:00000001:1.0:1775768300.210087:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768300.210087:0:36:0:(cl_io.c:314:cl_io_lock()) Process entered 02000000:00000001:1.0:1775768300.210090:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:3.0:1775768300.210091:0:36:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000100:00100000:2.0:1775768300.210093:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfed80 x1862022294299136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000010:1.0:1775768300.210094:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543e400. 00000020:00000001:3.0:1775768300.210094:0:36:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768300.210099:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768300.210100:0:36:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768300.210106:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:3.0:1775768300.210106:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775768300.210110:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c688. 00020000:00000001:3.0:1775768300.210111:0:36:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768300.210112:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000010:1.0:1775768300.210116:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6990. 00000100:00000001:2.0:1775768300.210116:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768300.210116:0:36:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:3.0:1775768300.210119:0:36:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775768300.210121:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294299008, portal 10 00000020:00000001:3.0:1775768300.210122:0:36:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768300.210125:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb9b9eae] to pc [ptlrpcd_rcv+-1] req@ffff994207dfed80 x1862022294299136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768300.210126:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000020:00000001:3.0:1775768300.210127:0:36:0:(cl_io.c:528:cl_io_start()) Process entered 00000100:00000001:1.0:1775768300.210129:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611840000 : -112966097711616 : ffff994207dfce00) 00020000:00000001:3.0:1775768300.210131:0:36:0:(lov_io.c:1114:lov_io_start()) Process entered 00020000:00000001:3.0:1775768300.210135:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775768300.210138:0:36:0:(cl_io.c:528:cl_io_start()) Process entered 00000100:00000040:1.0:1775768300.210139:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfce00 x1862022294299008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768325 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1775768300.210142:0:36:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000008:00000001:3.0:1775768300.210147:0:36:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000100:00000001:2.0:1775768300.210148:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768300.210176:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000008:00000001:3.0:1775768300.210180:0:36:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000100:00080000:2.0:1775768300.210181:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1775768300.210184:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000008:00000020:3.0:1775768300.210185:0:36:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000100:00080000:2.0:1775768300.210187:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000008:00000001:3.0:1775768300.210188:0:36:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768300.210193:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775768300.210194:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00000001:3.0:1775768300.210194:0:36:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000100:00080000:2.0:1775768300.210198:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5552 00000008:00000001:3.0:1775768300.210199:0:36:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000100:00000040:1.0:1775768300.210203:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000008:00000001:3.0:1775768300.210204:0:36:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107754940544 : -112965954611072 : ffff994210675880) 00000400:00000010:1.0:1775768300.210208:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6cc0. 00000400:00000200:2.0:1775768300.210209:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00000020:3.0:1775768300.210211:0:36:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994210675880 ready 0|-|- wr 1|-|+ rd 0|- 10 in flight 00000100:00000200:1.0:1775768300.210213:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294299008, offset 0 00000400:00000200:2.0:1775768300.210216:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000008:00000001:3.0:1775768300.210219:0:36:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000100:00080000:2.0:1775768300.210220:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000008:00000020:3.0:1775768300.210223:0:36:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000400:00000200:1.0:1775768300.210225:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000008:00000001:3.0:1775768300.210226:0:36:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768300.210227:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768300.210232:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000008:00000001:3.0:1775768300.210232:0:36:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000400:00000200:1.0:1775768300.210234:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00000020:3.0:1775768300.210236:0:36:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994218232fc0 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000100:00000040:2.0:1775768300.210238:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000200:1.0:1775768300.210242:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00000001:2.0:1775768300.210243:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:2.0:1775768300.210248:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000008:00000001:3.0:1775768300.210248:0:36:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1775768300.210249:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000040:2.0:1775768300.210251:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000008:00000001:3.0:1775768300.210253:0:36:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775768300.210254:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:1.0:1775768300.210255:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000020:00000040:2.0:1775768300.210258:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:3.0:1775768300.210259:0:36:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768300.210261:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000020:00000001:2.0:1775768300.210264:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00020000:00000001:3.0:1775768300.210264:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768300.210266:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000020:00000001:2.0:1775768300.210269:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00020000:00000001:3.0:1775768300.210270:0:36:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768300.210275:0:36:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768300.210276:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225509:225509:255:4294967295] 192.168.202.156@tcp LPNI seq info [225505:225505:7:4294967295] 00000100:00000040:2.0:1775768300.210276:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000001:3.0:1775768300.210280:0:36:0:(cl_io.c:555:cl_io_end()) Process entered 00000100:00000001:2.0:1775768300.210281:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00020000:00000001:3.0:1775768300.210285:0:36:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00000400:00000200:1.0:1775768300.210286:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000040:2.0:1775768300.210286:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00020000:00000001:3.0:1775768300.210288:0:36:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000100:00000001:2.0:1775768300.210291:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775768300.210291:0:36:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000040:2.0:1775768300.210296:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000020:00000001:3.0:1775768300.210296:0:36:0:(cl_io.c:563:cl_io_end()) Process leaving 00020000:00000001:3.0:1775768300.210299:0:36:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768300.210302:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000100:00080000:2.0:1775768300.210303:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00020000:00000001:3.0:1775768300.210305:0:36:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000020:00000001:3.0:1775768300.210309:0:36:0:(cl_io.c:563:cl_io_end()) Process leaving 00000100:00000001:2.0:1775768300.210310:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768300.210313:0:36:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000800:00000200:1.0:1775768300.210314:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 02000000:00000001:2.0:1775768300.210314:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00020000:00000001:3.0:1775768300.210317:0:36:0:(lov_io.c:1208:lov_io_unlock()) Process entered 02000000:00000001:2.0:1775768300.210318:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000010:1.0:1775768300.210319:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541e00. 00020000:00000001:3.0:1775768300.210320:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 02000000:00000001:2.0:1775768300.210323:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768300.210323:0:36:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000020:00000001:3.0:1775768300.210327:0:36:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000800:00000200:1.0:1775768300.210328:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:2.0:1775768300.210328:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00020000:00000001:3.0:1775768300.210330:0:36:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1775768300.210332:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00020000:00000001:3.0:1775768300.210335:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1775768300.210336:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768300.210340:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768300.210340:0:36:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000800:00000200:1.1:1775768300.210340:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000001:3.0:1775768300.210343:0:36:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000100:00000001:1.0:1775768300.210346:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768300.210347:0:36:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000100:00000010:2.0:1775768300.210348:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfe300. 00000020:00000001:3.0:1775768300.210350:0:36:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000040:2.0:1775768300.210352:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00000100:00000040:1.0:1775768300.210353:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000020:00000001:3.0:1775768300.210353:0:36:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:3.0:1775768300.210357:0:36:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 02000000:00000001:2.0:1775768300.210358:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00020000:00000001:3.0:1775768300.210359:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000400:00000200:1.0:1775768300.210361:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 02000000:00000001:2.0:1775768300.210362:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768300.210362:0:36:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000100:00000001:2.0:1775768300.210366:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000020:00000001:3.0:1775768300.210366:0:36:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000100:00000001:1.0:1775768300.210367:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300225043 02000000:00000001:2.0:1775768300.210370:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00020000:00000001:3.0:1775768300.210370:0:36:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768300.210373:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300226365 02000000:00000001:2.0:1775768300.210373:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768300.210374:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768300.210378:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300225043 00020000:00000001:3.0:1775768300.210379:0:36:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 02000000:00000010:2.0:1775768300.210380:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1f000. 00000100:00000001:1.0:1775768300.210383:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300226365 00000020:00000001:3.0:1775768300.210383:0:36:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000100:00000001:2.0:1775768300.210385:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768300.210386:0:36:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768300.210387:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768300.210392:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768300.210392:0:36:0:(cl_io.c:71:cl_io_fini()) Process entered 00000100:00100000:2.0:1775768300.210395:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfe300 x1862022294299264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00020000:00000001:3.0:1775768300.210396:0:36:0:(lov_io.c:675:lov_io_fini()) Process entered 00000100:00000001:1.0:1775768300.210397:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768300.210400:0:36:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000100:00000001:1.0:1775768300.210401:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1775768300.210403:0:36:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:3.0:1775768300.210407:0:36:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000100:00000001:1.0:1775768300.210408:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768300.210411:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000020:00001000:3.0:1775768300.210412:0:36:0:(cl_object.c:925:cl_env_put()) 1@ffff99420693ecf0 00000100:00000001:2.0:1775768300.210413:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775768300.210415:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768300.210417:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768300.210418:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000010:3.0:1775768300.210420:0:36:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210dce400. 00000100:00000001:1.0:1775768300.210421:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768300.210425:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c661fef] to pc [ptlrpcd_rcv+-1] req@ffff994207dfe300 x1862022294299264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775768300.210430:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfed80 x1862022294299136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1775768300.210430:0:36:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207981f60. 00000008:00000010:3.0:1775768300.210439:0:36:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994208b51650. 00000100:00000001:2.0:1775768300.210448:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768300.210449:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00020000:00000010:3.0:1775768300.210449:0:36:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994218000000. 00000100:00000001:1.0:1775768300.210454:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768300.210454:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00008000:00000010:3.0:1775768300.210456:0:36:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994207e0ade8. 02000000:00000001:1.0:1775768300.210459:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:2.0:1775768300.210462:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 02000000:00000001:1.0:1775768300.210463:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000080:00000010:3.0:1775768300.210464:0:36:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99420541b0a8. 00000100:00000001:2.0:1775768300.210466:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:1.0:1775768300.210468:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768300.210470:0:8:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000080:00000010:3.0:1775768300.210472:0:36:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077c1da8. 00000100:00100000:1.0:1775768300.210474:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294299136:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768300.210476:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775768300.210479:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000010:3.0:1775768300.210479:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcf600. 00000100:00080000:2.0:1775768300.210482:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5552 02000000:00000001:1.0:1775768300.210483:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768300.210486:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:3.0:1775768300.210487:0:36:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942086f8160. 02000000:00000001:1.0:1775768300.210490:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:2.0:1775768300.210491:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000010:3.0:1775768300.210494:0:36:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994207e8ed00. 02000000:00000010:1.0:1775768300.210495:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543c400. 00000400:00000200:2.0:1775768300.210496:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 02000000:00000001:1.0:1775768300.210500:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768300.210501:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00008000:00000010:3.0:1775768300.210501:0:36:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994209270ef0. 00000100:00000040:2.0:1775768300.210507:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000080:00000010:3.0:1775768300.210508:0:36:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942090e3960. 00000400:00000200:1.0:1775768300.210509:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768300.210513:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000010:3.0:1775768300.210514:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcec00. 00000400:00000010:1.0:1775768300.210515:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c2c0. 00000100:00000040:2.0:1775768300.210517:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000010:3.0:1775768300.210519:0:36:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff99420693ecf0. 00000100:00000001:2.0:1775768300.210522:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000010:1.0:1775768300.210524:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6550. 00020000:00000001:3.0:1775768300.210524:0:36:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00000020:00000001:2.0:1775768300.210526:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000200:1.0:1775768300.210529:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294299136, portal 10 00000020:00000040:2.0:1775768300.210529:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00020000:00000001:3.0:1775768300.210529:0:36:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000100:00000001:1.0:1775768300.210533:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000020:00000001:2.0:1775768300.210533:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000080:00000001:3.0:1775768300.210533:0:36:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000100:00000001:1.0:1775768300.210537:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611848064 : -112966097703552 : ffff994207dfed80) 00000020:00000040:2.0:1775768300.210538:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000080:00200000:3.0:1775768300.210538:0:36:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x10176:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000020:00000001:2.0:1775768300.210543:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000080:00000001:3.0:1775768300.210545:0:36:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:3.0:1775768300.210548:0:36:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000100:00000040:1.0:1775768300.210549:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfed80 x1862022294299136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768325 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1775768300.210549:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00001000:3.0:1775768300.210553:0:36:0:(cl_object.c:925:cl_env_put()) 1@ffff99420693e000 00000100:00000040:2.0:1775768300.210558:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000010:3.0:1775768300.210558:0:36:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210dce800. 00000100:00000001:1.0:1775768300.210564:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1775768300.210564:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000010:3.0:1775768300.210564:0:36:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207981900. 00000100:00000040:2.0:1775768300.210569:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000040:1.0:1775768300.210570:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000008:00000010:3.0:1775768300.210570:0:36:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994208b50000. 00000400:00000010:1.0:1775768300.210575:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6440. 00000100:00000001:2.0:1775768300.210575:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00020000:00000010:3.0:1775768300.210576:0:36:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994218000300. 00000100:00000200:1.0:1775768300.210579:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294299136, offset 0 00000020:00000040:2.0:1775768300.210581:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00008000:00000010:3.0:1775768300.210581:0:36:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994207e08e20. 00000400:00000200:1.0:1775768300.210587:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000080:00000010:3.0:1775768300.210587:0:36:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99420541b1f8. 00000100:00080000:2.0:1775768300.210588:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000080:00000010:3.0:1775768300.210593:0:36:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077c0248. 00000400:00000200:1.0:1775768300.210594:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768300.210594:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768300.210598:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000020:00000010:3.0:1775768300.210598:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcf800. 00000400:00000200:1.0:1775768300.210602:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 02000000:00000001:2.0:1775768300.210603:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00000010:3.0:1775768300.210603:0:36:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942086f8420. 02000000:00000001:2.0:1775768300.210609:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768300.210610:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00020000:00000010:3.0:1775768300.210610:0:36:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994207e8e400. 00000100:00000001:2.0:1775768300.210612:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:1.0:1775768300.210616:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000008:00100000:2.0:1775768300.210616:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00008000:00000010:3.0:1775768300.210616:0:36:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994209270a90. 00000008:00000001:2.0:1775768300.210622:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768300.210623:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000080:00000010:3.0:1775768300.210623:0:36:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942090e33c0. 00000100:00000001:2.0:1775768300.210626:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768300.210629:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000020:00000010:3.0:1775768300.210629:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcf000. 00000100:00000010:2.0:1775768300.210632:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207dfc700. 00000020:00000010:3.0:1775768300.210634:0:36:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff99420693e000. 00000400:00000200:1.0:1775768300.210637:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225510:225510:254:4294967295] 192.168.202.156@tcp LPNI seq info [225506:225506:6:4294967295] 00000020:00000040:2.0:1775768300.210637:0:8:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00000080:00000001:3.0:1775768300.210639:0:36:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1775768300.210642:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775768300.210645:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:3.0:1775768300.210646:0:36:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768300.210649:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768300.210650:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775768300.210653:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775768300.210656:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768300.210661:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 02000000:00000010:2.0:1775768300.210662:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994202d1e800. 00000800:00000200:1.0:1775768300.210668:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00000001:2.0:1775768300.210668:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1775768300.210674:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541800. 00000100:00100000:2.0:1775768300.210676:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207dfc700 x1862022294299392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.0:1775768300.210683:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:2.0:1775768300.210689:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000200:1.1:1775768300.210691:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:2.0:1775768300.210693:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768300.210695:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768300.210699:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a176dd1] to pc [ptlrpcd_rcv+-1] req@ffff994207dfc700 x1862022294299392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775768300.210701:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:1.0:1775768300.210708:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768300.210713:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300225035 00000100:00000001:1.0:1775768300.210718:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300226365 00000100:00000001:2.0:1775768300.210718:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768300.210723:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300225035 00000100:00000040:2.0:1775768300.210726:0:8:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5562) 00000100:00000001:1.0:1775768300.210727:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300226365 00000100:00000001:1.0:1775768300.210730:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768300.210732:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768300.210735:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768300.210736:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768300.210739:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768300.210741:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1775768300.210744:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768300.210745:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768300.210747:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768300.210751:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfe300 x1862022294299264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768300.210758:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768300.210760:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768300.210762:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768300.210764:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768300.210766:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768300.210769:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294299264:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768300.210773:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768300.210775:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768300.210777:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768300.210779:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768300.210783:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543c800. 02000000:00000001:1.0:1775768300.210785:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768300.210788:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768300.210792:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023cbb0. 00000400:00000010:1.0:1775768300.210796:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf63b8. 00000100:00000200:1.0:1775768300.210799:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294299264, portal 4 00000100:00000001:1.0:1775768300.210801:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768300.210803:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611845376 : -112966097706240 : ffff994207dfe300) 00000100:00000040:1.0:1775768300.210808:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfe300 x1862022294299264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768325 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768300.210815:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768300.210818:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768300.210820:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6aa0. 00000100:00000200:1.0:1775768300.210823:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294299264, offset 0 00000400:00000200:1.0:1775768300.210829:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768300.210834:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768300.210837:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768300.210840:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768300.210842:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768300.210846:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768300.210848:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768300.210851:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225511:225511:253:4294967295] 192.168.202.156@tcp LPNI seq info [225507:225507:5:4294967295] 00000400:00000200:1.0:1775768300.210855:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768300.210862:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768300.210866:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768300.210870:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541c00. 00000800:00000200:1.0:1775768300.210875:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768300.210879:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:1.0:1775768300.210882:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768300.210885:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:1.0:1775768300.210889:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768300.210890:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300225058 00000100:00000001:1.0:1775768300.210893:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300226365 00000100:00000001:1.0:1775768300.210896:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300225058 00000100:00000001:1.0:1775768300.210898:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300226365 00000100:00000001:1.0:1775768300.210901:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768300.210903:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768300.210905:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775768300.210908:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207dfc700 x1862022294299392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768300.210916:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768300.210917:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768300.210920:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775768300.210921:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768300.210923:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768300.210926:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294299392:192.168.202.156@tcp:8:kworker.0 00000100:00000001:1.0:1775768300.210929:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775768300.210930:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775768300.210931:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768300.210933:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775768300.210936:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff99420543dc00. 02000000:00000001:1.0:1775768300.210938:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768300.210941:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:1.0:1775768300.210944:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff99421023c000. 00000400:00000010:1.0:1775768300.210947:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6d48. 00000100:00000200:1.0:1775768300.210949:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294299392, portal 4 00000100:00000001:1.0:1775768300.210951:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775768300.210953:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107611838208 : -112966097713408 : ffff994207dfc700) 00000100:00000040:1.0:1775768300.210958:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207dfc700 x1862022294299392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768325 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768300.210964:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775768300.210967:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1775768300.210969:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994208bf6b28. 00000100:00000200:1.0:1775768300.210971:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294299392, offset 0 00000400:00000200:1.0:1775768300.210976:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1775768300.210980:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768300.210983:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1775768300.210986:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768300.210988:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768300.211003:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775768300.211006:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1775768300.211009:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225512:225512:252:4294967295] 192.168.202.156@tcp LPNI seq info [225508:225508:4:4294967295] 00000400:00000200:1.0:1775768300.211013:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768300.211018:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1775768300.211023:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1775768300.211026:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994210541400. 00000800:00000200:1.0:1775768300.211030:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:1.1:1775768300.211033:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:1.0:1775768300.211036:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768300.211039:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:1.0:1775768300.211042:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768300.211044:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300225061 00000100:00000001:1.0:1775768300.211046:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300226365 00000100:00000001:1.0:1775768300.211048:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300225061 00000100:00000001:1.0:1775768300.211050:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300226365 00000100:00000001:1.0:1775768300.211052:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775768300.211054:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768300.211057:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768300.211058:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768300.211061:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768300.211062:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1775768300.211064:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775768300.211065:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768300.211067:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768300.211068:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768300.273287:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768300.273294:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768300.273300:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768300.273302:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768300.273306:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768300.273308:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768300.273311:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768300.273312:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768300.273315:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768300.273316:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768300.273335:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768300.273339:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768300.273342:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768300.337286:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768300.337293:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768300.337295:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768300.337300:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768300.337304:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768300.337308:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768300.337311:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768300.337314:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768300.337317:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768301.233269:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768301.233277:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768301.233278:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768301.233284:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768301.233289:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768301.233294:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768301.233298:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768301.233300:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768301.233303:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768301.233351:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768301.233354:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768301.233355:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768301.233358:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775768301.233360:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768301.233362:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768301.233364:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768301.233365:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768301.233367:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768301.233408:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768301.233413:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768301.233415:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768301.233418:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768301.233420:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768301.233423:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768301.233425:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768301.233426:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768301.233428:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768301.297285:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768301.297290:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768301.297296:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768301.297297:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768301.297300:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768301.297301:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768301.297304:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768301.297305:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768301.297307:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768301.297307:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768301.297368:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768301.297371:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768301.297374:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768301.361297:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768301.361304:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768301.361305:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768301.361310:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768301.361313:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768301.361317:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768301.361319:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768301.361322:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768301.361324:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768301.553279:0:15036:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4300226365, at 4300226368 00000100:00000040:1.0:1775768301.553279:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4300226365, at 4300226368 00000100:00000010:0.0:1775768301.553295:0:15036:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cba3c0. 00000100:00000010:1.0:1775768301.553299:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff994207cba240. 00000100:00000001:1.0:1775768302.257280:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768302.257280:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768302.257288:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768302.257290:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768302.257292:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768302.257294:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768302.257295:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:1.0:1775768302.257298:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:3.0:1775768302.257299:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768302.257302:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768302.257304:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768302.257305:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768302.257309:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768302.257312:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768302.257314:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768302.257314:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768302.257317:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768302.257319:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768302.257360:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768302.257363:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768302.257364:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768302.257365:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768302.257367:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768302.257368:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768302.257370:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768302.257371:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768302.257372:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768302.321269:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768302.321278:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768302.321281:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768302.321284:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768302.321289:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768302.321292:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768302.321293:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768302.321295:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768302.321299:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768302.321301:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768302.321304:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768302.321305:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768302.321308:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768302.321309:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768302.321311:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768302.321312:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768302.321329:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768302.321331:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768302.321333:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768302.385278:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768302.385286:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768302.385288:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768302.385295:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768302.385300:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768302.385304:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768302.385307:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768302.385311:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768302.385314:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768303.281253:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768303.281254:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768303.281260:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768303.281261:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768303.281265:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768303.281265:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768303.281273:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775768303.281273:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768303.281278:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768303.281279:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768303.281285:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768303.281288:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768303.281290:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768303.281291:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768303.281292:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768303.281293:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768303.281299:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768303.281300:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768303.281302:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768303.281304:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768303.281308:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768303.281309:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768303.281316:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768303.281321:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768303.281323:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768303.281325:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768303.281327:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:1.0:1775768303.281522:0:2387:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.156:988 00000400:00000100:1.0:1775768303.281587:0:2387:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.156@tcp at host 192.168.202.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:1.0:1775768303.281600:0:2387:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768303.281606:0:2387:0:(router.c:1750:lnet_notify()) 192.168.202.56@tcp notifying 192.168.202.156@tcp: down 00000400:00000200:1.0:1775768303.281612:0:2387:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768303.281617:0:2387:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:1.0:1775768303.281623:0:2387:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000800:00000010:1.0:1775768303.281627:0:2387:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541300. 00000400:00000200:1.0:1775768303.281633:0:2387:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775768303.281639:0:2387:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775768303.281646:0:2387:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6bb0 00000400:00000010:1.0:1775768303.281649:0:2387:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6bb0. 00000100:00000001:1.0:1775768303.281656:0:2387:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768303.281670:0:2387:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff994207dff480 x1862022294298880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768325 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768303.281697:0:2387:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768303.281701:0:2387:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dff480 x1862022294298880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768325 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768303.281716:0:2387:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768303.281719:0:2387:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775768303.281732:0:2387:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000800:00000010:1.0:1775768303.281734:0:2387:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541e00. 00000400:00000200:1.0:1775768303.281741:0:2387:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775768303.281744:0:2387:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000100:00000001:2.0:1775768303.281746:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:1.0:1775768303.281750:0:2387:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6cc0 00000100:00000001:2.0:1775768303.281752:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000400:00000010:1.0:1775768303.281758:0:2387:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6cc0. 00000100:00000001:1.0:1775768303.281763:0:2387:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1775768303.281767:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000200:1.0:1775768303.281769:0:2387:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff994207dfce00 x1862022294299008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768325 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768303.281788:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768300/real 1775768303] req@ffff994207dff480 x1862022294298880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768303.281794:0:2387:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768303.281805:0:2387:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfce00 x1862022294299008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768325 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768303.281828:0:2387:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768303.281831:0:2387:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000001:2.0:1775768303.281832:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768303.281836:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768303.281843:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c738. 00000800:00000100:1.0:1775768303.281849:0:2387:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000800:00000010:1.0:1775768303.281855:0:2387:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541800. 00000400:00000200:1.0:1775768303.281859:0:2387:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775768303.281861:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf65d8 00000400:00000200:1.0:1775768303.281870:0:2387:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000400:00000010:2.0:1775768303.281872:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf65d8. 00000100:00000001:2.0:1775768303.281879:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000200:1.0:1775768303.281884:0:2387:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6440 00000100:00000200:2.0:1775768303.281887:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dff480 x1862022294298880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1775768303.281888:0:2387:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6440. 00000100:00000001:1.0:1775768303.281897:0:2387:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775768303.281907:0:2387:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff994207dfed80 x1862022294299136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768325 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768303.281918:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dff480 x1862022294298880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768303.281955:0:2387:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1775768303.281957:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000040:1.0:1775768303.281975:0:2387:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfed80 x1862022294299136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768325 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768303.281983:0:2387:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768303.281985:0:2387:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768303.281993:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768303.282000:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000800:00000100:1.0:1775768303.282001:0:2387:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000100:00000001:2.0:1775768303.282005:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:1.0:1775768303.282006:0:2387:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541c00. 00000400:00000200:1.0:1775768303.282010:0:2387:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775768303.282019:0:2387:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000100:00100000:2.0:1775768303.282019:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dff480 x1862022294298880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775768303.282029:0:2387:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6aa0 00000400:00000010:1.0:1775768303.282033:0:2387:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6aa0. 00000100:00000001:1.0:1775768303.282043:0:2387:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1775768303.282049:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:1.0:1775768303.282050:0:2387:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff994207dfe300 x1862022294299264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768325 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775768303.282066:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dff480 x1862022294298880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768325 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768303.282070:0:2387:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775768303.282077:0:2387:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfe300 x1862022294299264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768325 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768303.282113:0:2387:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768303.282116:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768303.282117:0:2387:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768303.282127:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000800:00000100:1.0:1775768303.282128:0:2387:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.156@tcp 00000800:00000010:1.0:1775768303.282139:0:2387:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994210541400. 00000100:00000001:2.0:1775768303.282139:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768303.282144:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1775768303.282147:0:2387:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000100:00000001:2.0:1775768303.282148:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768303.282206:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000400:00000200:1.0:1775768303.282216:0:2387:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_DROPPED 00000100:00080000:2.0:1775768303.282219:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000400:00000200:1.0:1775768303.282221:0:2387:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6b28 00000400:00000010:1.0:1775768303.282228:0:2387:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6b28. 00000100:00000001:1.0:1775768303.282234:0:2387:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:2.0:1775768303.282237:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768303.282244:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000200:1.0:1775768303.282246:0:2387:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff994207dfc700 x1862022294299392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768325 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768303.282261:0:2387:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768303.282268:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dff480 x1862022294298880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768325 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775768303.282269:0:2387:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207dfc700 x1862022294299392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768325 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768303.282347:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294298880:192.168.202.156@tcp:250:kworker.0 00000100:00000001:1.0:1775768303.282350:0:2387:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768303.282360:0:2387:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768303.282372:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768303.282376:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768303.282381:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dff480 x1862022294298880/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768325 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768303.282389:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768303.282395:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768303.282401:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543d800. 02000000:00000001:2.0:1775768303.282409:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768303.282412:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768303.282414:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:2.0:1775768303.282416:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768303.282419:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1d000. 02000000:00000001:2.0:1775768303.282423:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768303.282425:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768303.282427:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dff480. 00000100:00000001:2.0:1775768303.282430:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768303.282431:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768303.282433:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768303.282435:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768303.282436:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768303.282437:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768303.282441:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768300/real 1775768303] req@ffff994207dfe300 x1862022294299264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768303.282449:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768303.282451:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768303.282454:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023cbb0. 00000400:00000200:2.0:1775768303.282459:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf63b8 00000400:00000010:2.0:1775768303.282460:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf63b8. 00000100:00000001:2.0:1775768303.282463:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768303.282466:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfe300 x1862022294299264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768303.282472:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfe300 x1862022294299264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768303.282483:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768303.282489:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768303.282490:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768303.282491:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768303.282494:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfe300 x1862022294299264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768303.282500:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768303.282504:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfe300 x1862022294299264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768325 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768303.282511:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768303.282513:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768303.282515:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768303.282517:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768303.282519:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768303.282527:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768303.282529:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768303.282535:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768303.282541:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768303.282545:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfe300 x1862022294299264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768325 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768303.282551:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294299264:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768303.282555:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768303.282559:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768300/real 1775768303] req@ffff994207dfc700 x1862022294299392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768303.282566:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768303.282567:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768303.282570:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c000. 00000400:00000200:2.0:1775768303.282572:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6d48 00000400:00000010:2.0:1775768303.282574:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6d48. 00000100:00000001:2.0:1775768303.282576:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768303.282579:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfc700 x1862022294299392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768303.282586:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfc700 x1862022294299392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768303.282591:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768303.282592:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768303.282594:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768303.282595:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768303.282598:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfc700 x1862022294299392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768303.282603:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768303.282606:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfc700 x1862022294299392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768325 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768303.282611:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768303.282613:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768303.282615:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768303.282616:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768303.282617:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768303.282621:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768303.282624:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768303.282627:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768303.282629:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768303.282633:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfc700 x1862022294299392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768325 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768303.282639:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294299392:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768303.282644:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768303.282648:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768300/real 1775768303] req@ffff994207dfed80 x1862022294299136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768303.282656:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768303.282657:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768303.282659:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c2c0. 00000400:00000200:2.0:1775768303.282662:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6550 00000400:00000010:2.0:1775768303.282663:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6550. 00000100:00000001:2.0:1775768303.282665:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768303.282668:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfed80 x1862022294299136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768303.282675:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfed80 x1862022294299136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768303.282680:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768303.282681:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768303.282683:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768303.282684:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768303.282687:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfed80 x1862022294299136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768303.282692:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768303.282696:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfed80 x1862022294299136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768325 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768303.282702:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768303.282704:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768303.282705:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768303.282706:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768303.282708:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768303.282710:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768303.282712:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768303.282715:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768303.282717:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768303.282721:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfed80 x1862022294299136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768325 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768303.282728:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294299136:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768303.282731:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768303.282735:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768300/real 1775768303] req@ffff994207dfce00 x1862022294299008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768303.282742:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768303.282743:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768303.282744:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff99421023c688. 00000400:00000200:2.0:1775768303.282747:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994208bf6990 00000400:00000010:2.0:1775768303.282748:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994208bf6990. 00000100:00000001:2.0:1775768303.282750:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768303.282752:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207dfce00 x1862022294299008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768303.282758:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207dfce00 x1862022294299008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768303.282763:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768303.282764:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768303.282765:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768303.282767:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768303.282776:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207dfce00 x1862022294299008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768303.282781:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768303.282784:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207dfce00 x1862022294299008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768325 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768303.282794:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768303.282796:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768303.282797:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768303.282798:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768303.282800:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768303.282802:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768303.282804:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768303.282807:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768303.282810:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768303.282813:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207dfce00 x1862022294299008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768325 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768303.282820:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207dfce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294299008:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768303.282824:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768303.282826:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768303.282829:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfe300 x1862022294299264/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768325 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768303.282834:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768303.282836:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768303.282837:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543c800. 02000000:00000001:2.0:1775768303.282841:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768303.282842:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768303.282843:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:2.0:1775768303.282845:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768303.282846:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1f000. 02000000:00000001:2.0:1775768303.282849:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768303.282850:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768303.282851:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfe300. 00000100:00000001:2.0:1775768303.282853:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768303.282854:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768303.282856:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768303.282859:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfc700 x1862022294299392/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768325 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768303.282865:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768303.282866:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768303.282867:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543dc00. 02000000:00000001:2.0:1775768303.282869:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768303.282870:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768303.282872:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:2.0:1775768303.282873:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768303.282874:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1e800. 02000000:00000001:2.0:1775768303.282877:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768303.282878:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768303.282879:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfc700. 00000100:00000001:2.0:1775768303.282882:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768303.282883:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768303.282884:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768303.282887:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfed80 x1862022294299136/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768325 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768303.282893:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768303.282894:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768303.282895:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543c400. 02000000:00000001:2.0:1775768303.282897:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768303.282898:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768303.282899:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:2.0:1775768303.282900:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768303.282901:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1ec00. 02000000:00000001:2.0:1775768303.282903:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768303.282904:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768303.282905:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfed80. 00000100:00000001:2.0:1775768303.282907:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768303.282908:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768303.282909:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768303.282912:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207dfce00 x1862022294299008/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768325 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768303.282923:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768303.282925:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768303.282926:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff99420543e400. 02000000:00000001:2.0:1775768303.282929:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768303.282930:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768303.282931:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:2.0:1775768303.282932:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768303.282934:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994202d1fc00. 02000000:00000001:2.0:1775768303.282936:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768303.282937:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768303.282938:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207dfce00. 00000100:00000001:2.0:1775768303.282940:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768303.282942:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768303.282943:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768303.282950:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768303.282952:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768303.282956:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768303.282957:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768303.345294:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768303.345300:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768303.345305:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768303.345306:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768303.345309:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768303.345310:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768303.345313:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768303.345314:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768303.345316:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768303.345316:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768303.345369:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768303.345374:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768303.345377:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768303.409298:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768303.409306:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768303.409308:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768303.409315:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768303.409320:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768303.409324:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768303.409327:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768303.409329:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768303.409332:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768304.305285:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768304.305293:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768304.305296:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768304.305301:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768304.305306:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768304.305312:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768304.305317:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768304.305319:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768304.305321:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768304.305327:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768304.305327:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768304.305330:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768304.305337:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768304.305340:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768304.305344:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768304.305346:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768304.305348:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768304.305351:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768304.305469:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768304.305474:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768304.305475:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768304.305478:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768304.305481:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768304.305483:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768304.305485:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768304.305487:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768304.305489:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768304.369294:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768304.369300:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768304.369306:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768304.369307:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768304.369311:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768304.369313:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768304.369315:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768304.369316:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768304.369319:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768304.369320:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768304.369339:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768304.369342:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768304.369344:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768304.433250:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768304.433255:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768304.433257:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768304.433261:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768304.433263:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768304.433266:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768304.433268:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768304.433270:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768304.433272:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768305.329237:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768305.329238:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768305.329242:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768305.329244:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768305.329244:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768305.329245:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768305.329248:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768305.329249:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:2.0:1775768305.329250:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768305.329251:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768305.329253:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775768305.329255:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768305.329258:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768305.329261:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775768305.329262:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768305.329266:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768305.329266:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768305.329267:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768305.329270:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768305.329271:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768305.329276:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768305.329276:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768305.329278:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768305.329281:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768305.329282:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1775768305.329284:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775768305.329285:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768305.329290:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768305.329291:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775768305.329295:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768305.329297:0:8:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768305.329300:0:8:0:(import.c:484:import_select_connection()) Process entered 00000080:00000001:3.0:1775768305.329304:0:36:0:(rw.c:1481:ll_writepages()) Process entered 00000100:00080000:0.0:1775768305.329305:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5557 00000080:00000001:3.0:1775768305.329309:0:36:0:(file.c:5250:cl_sync_file_range()) Process entered 00000020:00000001:3.0:1775768305.329313:0:36:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000400:00000200:0.0:1775768305.329314:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768305.329319:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000020:00000010:3.0:1775768305.329319:0:36:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff99420693e000. 00000100:00080000:0.0:1775768305.329323:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768305.329326:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000010:3.0:1775768305.329327:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcea00. 00000100:00000001:0.0:1775768305.329328:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768305.329331:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768305.329333:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000010:3.0:1775768305.329333:0:36:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210dcf200. 00000020:00000001:0.0:1775768305.329337:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768305.329338:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000010:3.0:1775768305.329339:0:36:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207981900. 00000020:00000001:0.0:1775768305.329340:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768305.329344:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:0.0:1775768305.329345:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:0.0:1775768305.329347:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000008:00000010:3.0:1775768305.329348:0:36:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994208b50000. 00000100:00000040:0.0:1775768305.329349:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768305.329352:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768305.329353:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768305.329354:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00020000:00000010:3.0:1775768305.329354:0:36:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994218000300. 00000020:00000040:0.0:1775768305.329356:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00008000:00000010:3.0:1775768305.329361:0:36:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994207e08e20. 00000100:00080000:0.0:1775768305.329365:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768305.329366:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:3.0:1775768305.329368:0:36:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99420541b1f8. 02000000:00000001:0.0:1775768305.329369:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768305.329371:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768305.329373:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768305.329375:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000080:00000010:3.0:1775768305.329375:0:36:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077c0248. 00000100:00000001:0.0:1775768305.329377:0:8:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1775768305.329380:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcf000. 00000100:00000010:0.0:1775768305.329383:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fe680. 00000008:00000010:3.0:1775768305.329385:0:36:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942086f8420. 00000020:00000040:0.0:1775768305.329388:0:8:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 00020000:00000010:3.0:1775768305.329391:0:36:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994207e8e400. 02000000:00000001:0.0:1775768305.329393:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768305.329396:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00008000:00000010:3.0:1775768305.329396:0:36:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994209270a90. 00000100:00000001:0.0:1775768305.329401:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000080:00000010:3.0:1775768305.329402:0:36:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942090e33c0. 02000000:00000001:0.0:1775768305.329404:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000020:00000001:3.0:1775768305.329407:0:36:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107590086664 : -112966119464952 : ffff99420693e008) 02000000:00000001:0.0:1775768305.329410:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:3.0:1775768305.329411:0:36:0:(cl_object.c:831:cl_env_get()) 1@ffff99420693e000 00000020:00000001:3.0:1775768305.329417:0:36:0:(cl_object.c:151:cl_object_top()) ffff9942078a76c0 -> ffff9942078a76c0 00000020:00000001:3.0:1775768305.329420:0:36:0:(cl_io.c:125:__cl_io_init()) Process entered 02000000:00000010:0.0:1775768305.329421:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002b800. 00000080:00000001:3.0:1775768305.329423:0:36:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:3.0:1775768305.329426:0:36:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:3.0:1775768305.329430:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000100:00000001:0.0:1775768305.329431:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768305.329432:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:3.0:1775768305.329435:0:36:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:3.0:1775768305.329440:0:36:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017a:0x0]io ffff9942077c02a8 type 6 ignore/verify layout 1/0 00000100:00100000:0.0:1775768305.329442:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fe680 x1862022294299520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00020000:00000002:3.0:1775768305.329445:0:36:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207c1f9b0) owner 0000000000000000/ffff994201348000 00020000:00000001:3.0:1775768305.329451:0:36:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:3.0:1775768305.329454:0:36:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:3.0:1775768305.329456:0:36:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00020000:00000001:3.0:1775768305.329459:0:36:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768305.329463:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00020000:00000001:3.0:1775768305.329464:0:36:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:3.0:1775768305.329467:0:36:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00000100:00000001:0.0:1775768305.329470:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768305.329470:0:36:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768305.329474:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:3.0:1775768305.329475:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00000040:0.0:1775768305.329478:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ba41ba08] to pc [ptlrpcd_rcv+-1] req@ffff9942099fe680 x1862022294299520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:3.0:1775768305.329478:0:36:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00020000:00000002:3.0:1775768305.329480:0:36:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207c1f9b0) owner 0000000000000000/ffff994201348000 00000020:00000001:3.0:1775768305.329486:0:36:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768305.329492:0:36:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:3.0:1775768305.329495:0:36:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00020000:00000001:3.0:1775768305.329497:0:36:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00020000:00200000:3.0:1775768305.329500:0:36:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00000002:3.0:1775768305.329504:0:36:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:3.0:1775768305.329507:0:36:0:(lov_io.c:155:lov_sub_get()) Process entered 00020000:00000001:3.0:1775768305.329509:0:36:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000020:00000001:3.0:1775768305.329510:0:36:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00000001:0.0:1775768305.329513:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1775768305.329514:0:36:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff99420693ecf0. 00000100:00080000:0.0:1775768305.329518:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000010:3.0:1775768305.329519:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcf800. 00000100:00080000:0.0:1775768305.329525:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000020:00000010:3.0:1775768305.329525:0:36:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210dce800. 00000100:00000001:0.0:1775768305.329527:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000020:00000010:3.0:1775768305.329529:0:36:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207981f60. 00000100:00080000:0.0:1775768305.329530:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000008:00000010:3.0:1775768305.329536:0:36:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994208b51650. 00000100:00000001:0.0:1775768305.329541:0:8:0:(import.c:484:import_select_connection()) Process entered 00020000:00000010:3.0:1775768305.329541:0:36:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994218000000. 00000100:00080000:0.0:1775768305.329545:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5557 00008000:00000010:3.0:1775768305.329548:0:36:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994207e0ade8. 00000400:00000200:0.0:1775768305.329551:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000080:00000010:3.0:1775768305.329554:0:36:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99420541b0a8. 00000400:00000200:0.0:1775768305.329556:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768305.329560:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000080:00000010:3.0:1775768305.329560:0:36:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077c1da8. 00000100:00000040:0.0:1775768305.329565:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000010:3.0:1775768305.329567:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcec00. 00000100:00000001:0.0:1775768305.329568:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768305.329571:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000008:00000010:3.0:1775768305.329572:0:36:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942086f8160. 00000100:00000001:0.0:1775768305.329574:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768305.329578:0:8:0:(genops.c:811:class_conn2export()) Process entered 00020000:00000010:3.0:1775768305.329578:0:36:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994207e8ed00. 00000020:00000040:0.0:1775768305.329580:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:0.0:1775768305.329583:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00008000:00000010:3.0:1775768305.329583:0:36:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994209270ef0. 00000020:00000040:0.0:1775768305.329586:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000080:00000010:3.0:1775768305.329588:0:36:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942090e3960. 00000020:00000001:0.0:1775768305.329589:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:0.0:1775768305.329593:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:3.0:1775768305.329594:0:36:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107590089976 : -112966119461640 : ffff99420693ecf8) 00000100:00000040:0.0:1775768305.329597:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00001000:3.0:1775768305.329599:0:36:0:(cl_object.c:831:cl_env_get()) 1@ffff99420693ecf0 00000100:00000001:0.0:1775768305.329600:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768305.329603:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000001:3.0:1775768305.329605:0:36:0:(cl_object.c:151:cl_object_top()) ffff9942078a7788 -> ffff9942078a76c0 00000100:00000001:0.0:1775768305.329606:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775768305.329609:0:36:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000040:0.0:1775768305.329610:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 00000020:00000001:3.0:1775768305.329613:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000100:00080000:0.0:1775768305.329614:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000020:00000001:3.0:1775768305.329617:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000100:00000001:0.0:1775768305.329618:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768305.329621:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000020:00000001:3.0:1775768305.329622:0:36:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768305.329624:0:36:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768305.329627:0:36:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107612435560 : -112966097116056 : ffff994207e8e468) 02000000:00000001:0.0:1775768305.329628:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768305.329633:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768305.329635:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000020:00000001:3.0:1775768305.329635:0:36:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000020:00000001:3.0:1775768305.329638:0:36:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1775768305.329640:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00020000:00000001:3.0:1775768305.329642:0:36:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1775768305.329645:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768305.329646:0:36:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768305.329649:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768305.329650:0:36:0:(cl_io.c:314:cl_io_lock()) Process entered 00020000:00000001:3.0:1775768305.329653:0:36:0:(lov_io.c:1108:lov_io_lock()) Process entered 00000100:00000010:0.0:1775768305.329655:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fca80. 00020000:00000001:3.0:1775768305.329655:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775768305.329657:0:36:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000040:0.0:1775768305.329659:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:3.0:1775768305.329660:0:36:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:3.0:1775768305.329664:0:36:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768305.329665:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768305.329667:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768305.329668:0:36:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768305.329670:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00020000:00000001:3.0:1775768305.329672:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768305.329673:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00020000:00000001:3.0:1775768305.329676:0:36:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768305.329677:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768305.329679:0:36:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:3.0:1775768305.329683:0:36:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768305.329685:0:36:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768305.329686:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220028400. 00000020:00000001:3.0:1775768305.329687:0:36:0:(cl_io.c:528:cl_io_start()) Process entered 00000100:00000001:0.0:1775768305.329691:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768305.329692:0:36:0:(lov_io.c:1114:lov_io_start()) Process entered 00020000:00000001:3.0:1775768305.329695:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775768305.329696:0:36:0:(cl_io.c:528:cl_io_start()) Process entered 00000100:00100000:0.0:1775768305.329698:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fca80 x1862022294299648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000008:00000001:3.0:1775768305.329700:0:36:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000008:00000001:3.0:1775768305.329704:0:36:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000100:00000001:0.0:1775768305.329709:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000008:00000001:3.0:1775768305.329712:0:36:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000100:00000001:0.0:1775768305.329713:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000008:00000020:3.0:1775768305.329716:0:36:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000100:00000040:0.0:1775768305.329720:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000046a3ca2f] to pc [ptlrpcd_rcv+-1] req@ffff9942099fca80 x1862022294299648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1775768305.329720:0:36:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1775768305.329725:0:36:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000008:00000001:3.0:1775768305.329728:0:36:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000008:00000001:3.0:1775768305.329732:0:36:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107884691392 : -112965824860224 : ffff994218232fc0) 00000100:00000001:0.0:1775768305.329733:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768305.329737:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000008:00000020:3.0:1775768305.329739:0:36:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994218232fc0 ready 0|-|- wr 1|-|+ rd 0|- 10 in flight 00000100:00080000:0.0:1775768305.329743:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775768305.329747:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000008:00000001:3.0:1775768305.329747:0:36:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:3.0:1775768305.329750:0:36:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000100:00080000:0.0:1775768305.329751:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000008:00000001:3.0:1775768305.329754:0:36:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768305.329755:0:8:0:(import.c:484:import_select_connection()) Process entered 00000008:00000001:3.0:1775768305.329757:0:36:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000100:00080000:0.0:1775768305.329759:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5557 00000008:00000020:3.0:1775768305.329763:0:36:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994210675880 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000400:00000200:0.0:1775768305.329766:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00000001:3.0:1775768305.329770:0:36:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1775768305.329771:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768305.329775:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000008:00000001:3.0:1775768305.329776:0:36:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768305.329780:0:36:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768305.329781:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00020000:00000001:3.0:1775768305.329783:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768305.329785:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00020000:00000001:3.0:1775768305.329786:0:36:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768305.329789:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000001:3.0:1775768305.329790:0:36:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768305.329793:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775768305.329794:0:36:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000001:0.0:1775768305.329798:0:8:0:(genops.c:811:class_conn2export()) Process entered 00020000:00000001:3.0:1775768305.329798:0:36:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00000020:00000040:0.0:1775768305.329799:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00020000:00000001:3.0:1775768305.329801:0:36:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000020:00000001:0.0:1775768305.329802:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000001:3.0:1775768305.329803:0:36:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000040:0.0:1775768305.329805:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:3.0:1775768305.329806:0:36:0:(cl_io.c:563:cl_io_end()) Process leaving 00000020:00000001:0.0:1775768305.329809:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00020000:00000001:3.0:1775768305.329809:0:36:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768305.329814:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00020000:00000001:3.0:1775768305.329814:0:36:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000020:00000001:3.0:1775768305.329816:0:36:0:(cl_io.c:563:cl_io_end()) Process leaving 00000100:00000040:0.0:1775768305.329819:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000001:3.0:1775768305.329819:0:36:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000100:00000001:0.0:1775768305.329822:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00020000:00000001:3.0:1775768305.329823:0:36:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00000100:00000040:0.0:1775768305.329825:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00020000:00000001:3.0:1775768305.329826:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775768305.329830:0:36:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000100:00000001:0.0:1775768305.329831:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:3.0:1775768305.329832:0:36:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000040:0.0:1775768305.329834:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00020000:00000001:3.0:1775768305.329835:0:36:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768305.329840:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768305.329841:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00020000:00000001:3.0:1775768305.329845:0:36:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000100:00000001:0.0:1775768305.329846:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768305.329847:0:36:0:(cl_io.c:373:cl_io_unlock()) Process leaving 02000000:00000001:0.0:1775768305.329849:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000020:00000001:3.0:1775768305.329850:0:36:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 02000000:00000001:0.0:1775768305.329853:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1775768305.329853:0:36:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000001:3.0:1775768305.329855:0:36:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 02000000:00000001:0.0:1775768305.329857:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768305.329858:0:36:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00000100:00000001:0.0:1775768305.329860:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00020000:00000001:3.0:1775768305.329861:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775768305.329863:0:36:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000008:00100000:0.0:1775768305.329864:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000020:00000001:3.0:1775768305.329865:0:36:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000008:00000001:0.0:1775768305.329868:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768305.329869:0:36:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768305.329871:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768305.329873:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768305.329876:0:36:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000100:00000010:0.0:1775768305.329877:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fce00. 00000020:00000001:3.0:1775768305.329878:0:36:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000020:00000040:0.0:1775768305.329881:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:3.0:1775768305.329881:0:36:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768305.329886:0:36:0:(cl_io.c:71:cl_io_fini()) Process entered 02000000:00000001:0.0:1775768305.329887:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00020000:00000001:3.0:1775768305.329890:0:36:0:(lov_io.c:675:lov_io_fini()) Process entered 02000000:00000001:0.0:1775768305.329892:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768305.329893:0:36:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000100:00000001:0.0:1775768305.329895:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000020:00000001:3.0:1775768305.329896:0:36:0:(cl_io.c:71:cl_io_fini()) Process entered 02000000:00000001:0.0:1775768305.329898:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000020:00000001:3.0:1775768305.329899:0:36:0:(cl_io.c:112:cl_io_fini()) Process leaving 02000000:00000001:0.0:1775768305.329902:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:3.0:1775768305.329902:0:36:0:(cl_object.c:925:cl_env_put()) 1@ffff99420693ecf0 02000000:00000010:0.0:1775768305.329908:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002bc00. 00000020:00000010:3.0:1775768305.329910:0:36:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210dce800. 00000100:00000001:0.0:1775768305.329912:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768305.329918:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fce00 x1862022294299776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000010:3.0:1775768305.329918:0:36:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207981f60. 00000008:00000010:3.0:1775768305.329926:0:36:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994208b51650. 00000100:00000001:0.0:1775768305.329927:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768305.329931:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:3.0:1775768305.329932:0:36:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994218000000. 00000100:00000040:0.0:1775768305.329935:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000060e5e9d1] to pc [ptlrpcd_rcv+-1] req@ffff9942099fce00 x1862022294299776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00008000:00000010:3.0:1775768305.329937:0:36:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994207e0ade8. 00000100:00000001:0.0:1775768305.329959:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:3.0:1775768305.329959:0:36:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99420541b0a8. 00000100:00080000:0.0:1775768305.329963:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000080:00000010:3.0:1775768305.329965:0:36:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077c1da8. 00000100:00080000:0.0:1775768305.329968:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775768305.329971:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000020:00000010:3.0:1775768305.329971:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcf800. 00000100:00080000:0.0:1775768305.329972:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768305.329977:0:8:0:(import.c:484:import_select_connection()) Process entered 00000008:00000010:3.0:1775768305.329977:0:36:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942086f8160. 00000100:00080000:0.0:1775768305.329981:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5557 00020000:00000010:3.0:1775768305.329983:0:36:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994207e8ed00. 00000400:00000200:0.0:1775768305.329987:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768305.329990:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768305.329994:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00008000:00000010:3.0:1775768305.329996:0:36:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994209270ef0. 00000100:00000040:0.0:1775768305.329998:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768305.330001:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000080:00000010:3.0:1775768305.330001:0:36:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942090e3960. 00000100:00000040:0.0:1775768305.330005:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000020:00000010:3.0:1775768305.330006:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcec00. 00000100:00000001:0.0:1775768305.330010:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000010:3.0:1775768305.330011:0:36:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff99420693ecf0. 00000020:00000001:0.0:1775768305.330013:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768305.330016:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00020000:00000001:3.0:1775768305.330016:0:36:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00000020:00000001:0.0:1775768305.330017:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768305.330020:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000020:00000001:0.0:1775768305.330023:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:0.0:1775768305.330025:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:0.0:1775768305.330028:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768305.330030:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768305.330032:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768305.330039:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768305.330041:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00080000:0.0:1775768305.330043:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768305.330045:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768305.330047:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768305.330050:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768305.330051:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768305.330054:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775768305.330056:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:0.0:1775768305.330059:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768305.330061:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768305.330067:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fdc00. 00000020:00000040:0.0:1775768305.330069:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 02000000:00000001:0.0:1775768305.330072:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768305.330073:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768305.330075:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768305.330079:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768305.330082:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768305.330087:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220029800. 00000100:00000001:0.0:1775768305.330091:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768305.330092:0:36:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:3.0:1775768305.330096:0:36:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000100:00100000:0.0:1775768305.330097:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fdc00 x1862022294299904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000080:00200000:3.0:1775768305.330100:0:36:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:3.0:1775768305.330108:0:36:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:3.0:1775768305.330110:0:36:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000100:00000001:0.0:1775768305.330113:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000020:00001000:3.0:1775768305.330115:0:36:0:(cl_object.c:925:cl_env_put()) 1@ffff99420693e000 00000100:00000001:0.0:1775768305.330116:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1775768305.330119:0:36:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210dcf200. 00000100:00000040:0.0:1775768305.330123:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001372d9ae] to pc [ptlrpcd_rcv+-1] req@ffff9942099fdc00 x1862022294299904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1775768305.330125:0:36:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207981900. 00000008:00000010:3.0:1775768305.330132:0:36:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994208b50000. 00000100:00000001:0.0:1775768305.330137:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:3.0:1775768305.330137:0:36:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994218000300. 00008000:00000010:3.0:1775768305.330143:0:36:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994207e08e20. 00000100:00080000:0.0:1775768305.330144:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768305.330149:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775768305.330150:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000080:00000010:3.0:1775768305.330150:0:36:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99420541b1f8. 00000100:00080000:0.0:1775768305.330153:0:8:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000080:00000010:3.0:1775768305.330189:0:36:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077c0248. 00000020:00000010:3.0:1775768305.330193:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcea00. 00000008:00000010:3.0:1775768305.330195:0:36:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942086f8420. 00020000:00000010:3.0:1775768305.330198:0:36:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994207e8e400. 00008000:00000010:3.0:1775768305.330200:0:36:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994209270a90. 00000100:00000001:0.0:1775768305.330205:0:8:0:(import.c:484:import_select_connection()) Process entered 00000080:00000010:3.0:1775768305.330205:0:36:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942090e33c0. 00000020:00000010:3.0:1775768305.330209:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcf000. 00000100:00080000:0.0:1775768305.330210:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5557 00000020:00000010:3.0:1775768305.330213:0:36:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff99420693e000. 00000400:00000200:0.0:1775768305.330216:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000080:00000001:3.0:1775768305.330217:0:36:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:3.0:1775768305.330221:0:36:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768305.330223:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768305.330226:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768305.330229:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000080:00000001:3.0:1775768305.330230:0:36:0:(rw.c:1481:ll_writepages()) Process entered 00000100:00000001:0.0:1775768305.330232:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000080:00000001:3.0:1775768305.330233:0:36:0:(file.c:5250:cl_sync_file_range()) Process entered 00000020:00000001:3.0:1775768305.330235:0:36:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00000040:0.0:1775768305.330236:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768305.330239:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000010:3.0:1775768305.330240:0:36:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff99420693e000. 00000020:00000001:0.0:1775768305.330241:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768305.330243:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000010:3.0:1775768305.330244:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcf000. 00000020:00000001:0.0:1775768305.330245:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768305.330248:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000010:3.0:1775768305.330249:0:36:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210dcea00. 00000020:00000001:0.0:1775768305.330251:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000010:3.0:1775768305.330254:0:36:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207981900. 00000020:00000001:0.0:1775768305.330255:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:0.0:1775768305.330260:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000008:00000010:3.0:1775768305.330260:0:36:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994208b50000. 00000100:00000001:0.0:1775768305.330262:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00020000:00000010:3.0:1775768305.330265:0:36:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994218000300. 00000100:00000040:0.0:1775768305.330267:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00008000:00000010:3.0:1775768305.330269:0:36:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994207e08e20. 00000100:00000001:0.0:1775768305.330270:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768305.330275:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000080:00000010:3.0:1775768305.330275:0:36:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99420541b1f8. 00000100:00080000:0.0:1775768305.330279:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000080:00000010:3.0:1775768305.330280:0:36:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077c0248. 00000100:00000001:0.0:1775768305.330282:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1775768305.330284:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcf200. 02000000:00000001:0.0:1775768305.330285:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768305.330290:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00000010:3.0:1775768305.330290:0:36:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942086f8420. 02000000:00000001:0.0:1775768305.330294:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:3.0:1775768305.330295:0:36:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994207e8e400. 00000100:00000001:0.0:1775768305.330297:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00008000:00000010:3.0:1775768305.330299:0:36:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994209270a90. 00000008:00100000:0.0:1775768305.330300:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000080:00000010:3.0:1775768305.330303:0:36:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942090e33c0. 00000008:00000001:0.0:1775768305.330304:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768305.330308:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768305.330308:0:36:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107590086664 : -112966119464952 : ffff99420693e008) 00000020:00001000:3.0:1775768305.330313:0:36:0:(cl_object.c:831:cl_env_get()) 1@ffff99420693e000 00000100:00000010:0.0:1775768305.330314:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fea00. 00000020:00000001:3.0:1775768305.330315:0:36:0:(cl_object.c:151:cl_object_top()) ffff9942078a7f58 -> ffff9942078a7f58 00000020:00000040:0.0:1775768305.330318:0:8:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:3.0:1775768305.330319:0:36:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:3.0:1775768305.330321:0:36:0:(vvp_io.c:1812:vvp_io_init()) Process entered 02000000:00000001:0.0:1775768305.330323:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000080:00200000:3.0:1775768305.330324:0:36:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017b:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 02000000:00000001:0.0:1775768305.330325:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768305.330329:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000020:00000001:3.0:1775768305.330330:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 02000000:00000001:0.0:1775768305.330332:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000020:00000001:3.0:1775768305.330333:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 02000000:00000001:0.0:1775768305.330335:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:3.0:1775768305.330335:0:36:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:3.0:1775768305.330340:0:36:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017b:0x0]io ffff9942077c02a8 type 6 ignore/verify layout 1/0 02000000:00000010:0.0:1775768305.330341:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220029400. 00000100:00000001:0.0:1775768305.330346:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:3.0:1775768305.330346:0:36:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207c1f000) owner 0000000000000000/ffff994201348000 00020000:00000001:3.0:1775768305.330350:0:36:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00000100:00100000:0.0:1775768305.330353:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fea00 x1862022294300032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00020000:00000001:3.0:1775768305.330353:0:36:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:3.0:1775768305.330356:0:36:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00020000:00000001:3.0:1775768305.330359:0:36:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768305.330363:0:36:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768305.330366:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00020000:00000001:3.0:1775768305.330367:0:36:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00000100:00000001:0.0:1775768305.330370:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768305.330370:0:36:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768305.330375:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000100:00000040:0.0:1775768305.330377:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008d4eaacc] to pc [ptlrpcd_rcv+-1] req@ffff9942099fea00 x1862022294300032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775768305.330377:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:3.0:1775768305.330380:0:36:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00020000:00000002:3.0:1775768305.330383:0:36:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207c1f000) owner 0000000000000000/ffff994201348000 00000020:00000001:3.0:1775768305.330388:0:36:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768305.330393:0:36:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:3.0:1775768305.330394:0:36:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000100:00000001:0.0:1775768305.330395:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768305.330397:0:36:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00020000:00200000:3.0:1775768305.330400:0:36:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00000040:0.0:1775768305.330401:0:8:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5567) 00020000:00000002:3.0:1775768305.330404:0:36:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:3.0:1775768305.330408:0:36:0:(lov_io.c:155:lov_sub_get()) Process entered 00020000:00000001:3.0:1775768305.330409:0:36:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000020:00000001:3.0:1775768305.330410:0:36:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:3.0:1775768305.330413:0:36:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff99420693ecf0. 00000020:00000010:3.0:1775768305.330416:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcec00. 00000020:00000010:3.0:1775768305.330419:0:36:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210dcf800. 00000020:00000010:3.0:1775768305.330421:0:36:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207981f60. 00000008:00000010:3.0:1775768305.330424:0:36:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994208b51650. 00020000:00000010:3.0:1775768305.330426:0:36:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994218000000. 00008000:00000010:3.0:1775768305.330429:0:36:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994207e0ade8. 00000080:00000010:3.0:1775768305.330432:0:36:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99420541b0a8. 00000080:00000010:3.0:1775768305.330435:0:36:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077c1da8. 00000020:00000010:3.0:1775768305.330438:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dce800. 00000008:00000010:3.0:1775768305.330440:0:36:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942086f8160. 00020000:00000010:3.0:1775768305.330443:0:36:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994207e8ed00. 00008000:00000010:3.0:1775768305.330445:0:36:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994209270ef0. 00000080:00000010:3.0:1775768305.330448:0:36:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942090e3960. 00000020:00000001:3.0:1775768305.330450:0:36:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107590089976 : -112966119461640 : ffff99420693ecf8) 00000020:00001000:3.0:1775768305.330452:0:36:0:(cl_object.c:831:cl_env_get()) 1@ffff99420693ecf0 00000020:00000001:3.0:1775768305.330454:0:36:0:(cl_object.c:151:cl_object_top()) ffff9942078a75f8 -> ffff9942078a7f58 00000020:00000001:3.0:1775768305.330455:0:36:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:3.0:1775768305.330457:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:3.0:1775768305.330458:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:3.0:1775768305.330459:0:36:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768305.330461:0:36:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768305.330462:0:36:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107612435560 : -112966097116056 : ffff994207e8e468) 00000020:00000001:3.0:1775768305.330464:0:36:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000020:00000001:3.0:1775768305.330465:0:36:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768305.330467:0:36:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768305.330469:0:36:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768305.330470:0:36:0:(cl_io.c:314:cl_io_lock()) Process entered 00020000:00000001:3.0:1775768305.330471:0:36:0:(lov_io.c:1108:lov_io_lock()) Process entered 00020000:00000001:3.0:1775768305.330472:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775768305.330473:0:36:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:3.0:1775768305.330474:0:36:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:3.0:1775768305.330476:0:36:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768305.330477:0:36:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768305.330478:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768305.330480:0:36:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768305.330482:0:36:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:3.0:1775768305.330483:0:36:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768305.330484:0:36:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768305.330486:0:36:0:(cl_io.c:528:cl_io_start()) Process entered 00020000:00000001:3.0:1775768305.330488:0:36:0:(lov_io.c:1114:lov_io_start()) Process entered 00020000:00000001:3.0:1775768305.330489:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775768305.330490:0:36:0:(cl_io.c:528:cl_io_start()) Process entered 00000008:00000001:3.0:1775768305.330492:0:36:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000008:00000001:3.0:1775768305.330493:0:36:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000008:00000001:3.0:1775768305.330497:0:36:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:3.0:1775768305.330499:0:36:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000100:00000001:2.0:1775768305.330501:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00000001:3.0:1775768305.330501:0:36:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1775768305.330505:0:36:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000100:00000001:2.0:1775768305.330506:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000008:00000001:3.0:1775768305.330507:0:36:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000100:00000001:2.0:1775768305.330509:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000008:00000001:3.0:1775768305.330510:0:36:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107884688256 : -112965824863360 : ffff994218232380) 00000008:00000020:3.0:1775768305.330514:0:36:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994218232380 ready 0|-|- wr 1|-|+ rd 0|- 9 in flight 00000100:00000040:2.0:1775768305.330518:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fe680 x1862022294299520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1775768305.330518:0:36:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:3.0:1775768305.330521:0:36:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:3.0:1775768305.330523:0:36:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1775768305.330524:0:36:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000008:00000020:3.0:1775768305.330526:0:36:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994210674fc0 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000008:00000001:3.0:1775768305.330535:0:36:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768305.330540:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000008:00000001:3.0:1775768305.330540:0:36:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768305.330543:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768305.330544:0:36:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768305.330547:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768305.330548:0:36:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768305.330549:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:3.0:1775768305.330550:0:36:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768305.330552:0:36:0:(cl_io.c:555:cl_io_end()) Process entered 02000000:00000001:2.0:1775768305.330553:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00020000:00000001:3.0:1775768305.330554:0:36:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00020000:00000001:3.0:1775768305.330555:0:36:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 02000000:00000001:2.0:1775768305.330558:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768305.330558:0:36:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000001:3.0:1775768305.330559:0:36:0:(cl_io.c:563:cl_io_end()) Process leaving 00020000:00000001:3.0:1775768305.330562:0:36:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768305.330565:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294299520:192.168.202.156@tcp:250:kworker.0 00020000:00000001:3.0:1775768305.330566:0:36:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 00000020:00000001:3.0:1775768305.330568:0:36:0:(cl_io.c:563:cl_io_end()) Process leaving 00000020:00000001:3.0:1775768305.330571:0:36:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000100:00000001:2.0:1775768305.330572:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00020000:00000001:3.0:1775768305.330573:0:36:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00020000:00000001:3.0:1775768305.330576:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775768305.330577:0:36:0:(cl_io.c:352:cl_io_unlock()) Process entered 02000000:00000001:2.0:1775768305.330578:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:3.0:1775768305.330578:0:36:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:3.0:1775768305.330580:0:36:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768305.330582:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768305.330583:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768305.330586:0:36:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 02000000:00000001:2.0:1775768305.330588:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:3.0:1775768305.330588:0:36:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:3.0:1775768305.330589:0:36:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000020:00000001:3.0:1775768305.330591:0:36:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000001:3.0:1775768305.330592:0:36:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:3.0:1775768305.330593:0:36:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00020000:00000001:3.0:1775768305.330594:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 02000000:00000010:2.0:1775768305.330595:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1fc00. 00000020:00000001:3.0:1775768305.330595:0:36:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000001:3.0:1775768305.330598:0:36:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 02000000:00000001:2.0:1775768305.330599:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768305.330600:0:36:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768305.330603:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768305.330605:0:36:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000400:00000200:2.0:1775768305.330607:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:3.0:1775768305.330607:0:36:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000020:00000001:3.0:1775768305.330610:0:36:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775768305.330612:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000020:00000001:3.0:1775768305.330613:0:36:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:3.0:1775768305.330616:0:36:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:3.0:1775768305.330618:0:36:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:3.0:1775768305.330619:0:36:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:3.0:1775768305.330620:0:36:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000400:00000010:2.0:1775768305.330621:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000020:00001000:3.0:1775768305.330622:0:36:0:(cl_object.c:925:cl_env_put()) 1@ffff99420693ecf0 00000020:00000010:3.0:1775768305.330627:0:36:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210dcf800. 00000100:00000200:2.0:1775768305.330631:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294299520, portal 25 00000020:00000010:3.0:1775768305.330631:0:36:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207981f60. 00000008:00000010:3.0:1775768305.330635:0:36:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994208b51650. 00000100:00000001:2.0:1775768305.330636:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768305.330640:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641206400 : -112966068345216 : ffff9942099fe680) 00020000:00000010:3.0:1775768305.330640:0:36:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994218000000. 00008000:00000010:3.0:1775768305.330645:0:36:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994207e0ade8. 00000100:00000040:2.0:1775768305.330647:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fe680 x1862022294299520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768330 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000010:3.0:1775768305.330649:0:36:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99420541b0a8. 00000080:00000010:3.0:1775768305.330654:0:36:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077c1da8. 00000020:00000010:3.0:1775768305.330659:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcec00. 00000100:00000001:2.0:1775768305.330660:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00000010:3.0:1775768305.330664:0:36:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942086f8160. 00000100:00000040:2.0:1775768305.330665:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00020000:00000010:3.0:1775768305.330669:0:36:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994207e8ed00. 00000400:00000010:2.0:1775768305.330670:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00008000:00000010:3.0:1775768305.330674:0:36:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994209270ef0. 00000100:00000200:2.0:1775768305.330676:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294299520, offset 0 00000080:00000010:3.0:1775768305.330677:0:36:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942090e3960. 00000020:00000010:3.0:1775768305.330681:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dce800. 00000400:00000200:2.0:1775768305.330685:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000020:00000010:3.0:1775768305.330686:0:36:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff99420693ecf0. 00020000:00000001:3.0:1775768305.330690:0:36:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:3.0:1775768305.330692:0:36:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:3.0:1775768305.330693:0:36:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000400:00000200:2.0:1775768305.330696:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000080:00200000:3.0:1775768305.330696:0:36:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:3.0:1775768305.330700:0:36:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:3.0:1775768305.330702:0:36:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000400:00000200:2.0:1775768305.330704:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000020:00001000:3.0:1775768305.330705:0:36:0:(cl_object.c:925:cl_env_put()) 1@ffff99420693e000 00000020:00000010:3.0:1775768305.330709:0:36:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210dcea00. 00000020:00000010:3.0:1775768305.330712:0:36:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207981900. 00000400:00000200:2.0:1775768305.330713:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000008:00000010:3.0:1775768305.330715:0:36:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994208b50000. 00020000:00000010:3.0:1775768305.330719:0:36:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994218000300. 00000400:00000200:2.0:1775768305.330720:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00008000:00000010:3.0:1775768305.330722:0:36:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994207e08e20. 00000400:00000200:2.0:1775768305.330726:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000080:00000010:3.0:1775768305.330727:0:36:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99420541b1f8. 00000400:00000200:2.0:1775768305.330731:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000080:00000010:3.0:1775768305.330732:0:36:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077c0248. 00000020:00000010:3.0:1775768305.330736:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcf000. 00000400:00000200:2.0:1775768305.330739:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225513:225513:256:4294967295] 192.168.202.156@tcp LPNI seq info [225509:225509:8:4294967295] 00000008:00000010:3.0:1775768305.330740:0:36:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942086f8420. 00020000:00000010:3.0:1775768305.330745:0:36:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994207e8e400. 00008000:00000010:3.0:1775768305.330748:0:36:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994209270a90. 00000400:00000200:2.0:1775768305.330749:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000080:00000010:3.0:1775768305.330752:0:36:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942090e33c0. 00000020:00000010:3.0:1775768305.330756:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcf200. 00000020:00000010:3.0:1775768305.330758:0:36:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff99420693e000. 00000080:00000001:3.0:1775768305.330762:0:36:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:3.0:1775768305.330766:0:36:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768305.330767:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000080:00000001:3.0:1775768305.330773:0:36:0:(rw.c:1481:ll_writepages()) Process entered 00000080:00000001:3.0:1775768305.330775:0:36:0:(file.c:5250:cl_sync_file_range()) Process entered 00000800:00000200:2.0:1775768305.330777:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000020:00000001:3.0:1775768305.330778:0:36:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:3.0:1775768305.330782:0:36:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff99420693e000. 00000800:00000010:2.0:1775768305.330784:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000020:00000010:3.0:1775768305.330786:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcf200. 00000020:00000010:3.0:1775768305.330790:0:36:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210dcf000. 00000800:00000200:2.0:1775768305.330793:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000010:3.0:1775768305.330793:0:36:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207981900. 00000008:00000010:3.0:1775768305.330798:0:36:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994208b50000. 00000800:00000200:2.0:1775768305.330799:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00020000:00000010:3.0:1775768305.330802:0:36:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994218000300. 00008000:00000010:3.0:1775768305.330806:0:36:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994207e08e20. 00000800:00000200:2.1:1775768305.330808:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000080:00000010:3.0:1775768305.330811:0:36:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99420541b1f8. 00000800:00000200:2.1:1775768305.330812:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000080:00000010:3.0:1775768305.330816:0:36:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077c0248. 00000800:00000100:2.0:1775768305.330820:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000020:00000010:3.0:1775768305.330821:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcea00. 00000800:00000010:2.0:1775768305.330824:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000008:00000010:3.0:1775768305.330825:0:36:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942086f8420. 00000400:00000200:2.0:1775768305.330830:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00020000:00000010:3.0:1775768305.330830:0:36:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994207e8e400. 00008000:00000010:3.0:1775768305.330832:0:36:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994209270a90. 00000080:00000010:3.0:1775768305.330836:0:36:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942090e33c0. 00000400:00000200:2.0:1775768305.330837:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000020:00000001:3.0:1775768305.330840:0:36:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107590086664 : -112966119464952 : ffff99420693e008) 00000400:00000200:2.0:1775768305.330844:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000020:00001000:3.0:1775768305.330844:0:36:0:(cl_object.c:831:cl_env_get()) 1@ffff99420693e000 00000020:00000001:3.0:1775768305.330847:0:36:0:(cl_object.c:151:cl_object_top()) ffff99420791a850 -> ffff99420791a850 00000400:00000010:2.0:1775768305.330848:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000020:00000001:3.0:1775768305.330852:0:36:0:(cl_io.c:125:__cl_io_init()) Process entered 00000100:00000001:2.0:1775768305.330854:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000080:00000001:3.0:1775768305.330854:0:36:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:3.0:1775768305.330858:0:36:0:(vvp_io.c:1817:vvp_io_init()) [0x240000402:0x1017f:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000100:00000200:2.0:1775768305.330861:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fe680 x1862022294299520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768330 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775768305.330862:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:3.0:1775768305.330864:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:3.0:1775768305.330867:0:36:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:3.0:1775768305.330871:0:36:0:(lov_object.c:1554:lov_io_init()) [0x240000402:0x1017f:0x0]io ffff9942077c02a8 type 6 ignore/verify layout 1/0 00000100:00000001:2.0:1775768305.330875:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00020000:00000002:3.0:1775768305.330875:0:36:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff994207dfbd90) owner 0000000000000000/ffff994201348000 00000100:00000040:2.0:1775768305.330880:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fe680 x1862022294299520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768330 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:3.0:1775768305.330880:0:36:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:3.0:1775768305.330882:0:36:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:3.0:1775768305.330885:0:36:0:(lov_io.c:322:lov_io_mirror_init()) Process entered 00020000:00000001:3.0:1775768305.330888:0:36:0:(lov_io.c:333:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768305.330893:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768305.330893:0:36:0:(lov_io.c:611:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768305.330897:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00020000:00000001:3.0:1775768305.330898:0:36:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:3.0:1775768305.330900:0:36:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768305.330902:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768305.330902:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:3.0:1775768305.330903:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:3.0:1775768305.330906:0:36:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00020000:00000002:3.0:1775768305.330908:0:36:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff994207dfbd90) owner 0000000000000000/ffff994201348000 00000400:00000200:2.0:1775768305.330909:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:3.0:1775768305.330913:0:36:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768305.330914:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4300231486 00000020:00000001:3.0:1775768305.330917:0:36:0:(cl_io.c:823:cl_io_loop()) Process entered 00000100:00000010:2.0:1775768305.330919:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d6c00. 00000020:00000001:3.0:1775768305.330919:0:36:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00020000:00000001:3.0:1775768305.330920:0:36:0:(lov_io.c:836:lov_io_iter_init()) Process entered 00020000:00200000:3.0:1775768305.330923:0:36:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00000002:3.0:1775768305.330926:0:36:0:(lov_offset.c:247:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00000100:00000001:2.0:1775768305.330928:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00020000:00000001:3.0:1775768305.330929:0:36:0:(lov_io.c:155:lov_sub_get()) Process entered 00020000:00000001:3.0:1775768305.330932:0:36:0:(lov_io.c:104:lov_io_sub_init()) Process entered 00000020:00000001:3.0:1775768305.330933:0:36:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00000001:2.0:1775768305.330935:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4300231486 00000020:00000010:3.0:1775768305.330936:0:36:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff99420693ecf0. 00000100:00000010:2.0:1775768305.330952:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d6900. 00000020:00000010:3.0:1775768305.330953:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dce800. 00000100:00000001:2.0:1775768305.330957:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000020:00000010:3.0:1775768305.330957:0:36:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff994210dcec00. 00000100:00000001:2.0:1775768305.330963:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1775768305.330963:0:36:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff994207981f60. 00000008:00000010:3.0:1775768305.330967:0:36:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff994208b51650. 00000100:00000001:2.0:1775768305.330968:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:3.0:1775768305.330972:0:36:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff994218000000. 00000100:00000001:2.0:1775768305.330973:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00008000:00000010:3.0:1775768305.330977:0:36:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff994207e0ade8. 00000100:00100000:2.0:1775768305.330979:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768305/real 1775768305] req@ffff9942099fe680 x1862022294299520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000010:3.0:1775768305.330981:0:36:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff99420541b0a8. 00000080:00000010:3.0:1775768305.330987:0:36:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff9942077c1da8. 00000020:00000010:3.0:1775768305.330991:0:36:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff994210dcf800. 00000400:00000001:2.0:1775768305.330992:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768305.330995:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000008:00000010:3.0:1775768305.330995:0:36:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff9942086f8160. 00000400:00000010:2.0:1775768305.330997:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00020000:00000010:3.0:1775768305.330999:0:36:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff994207e8ed00. 00000400:00000200:2.0:1775768305.331001:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768305.331004:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00008000:00000010:3.0:1775768305.331004:0:36:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff994209270ef0. 00000100:00000001:2.0:1775768305.331009:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000080:00000010:3.0:1775768305.331010:0:36:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff9942090e3960. 00000100:00000200:2.0:1775768305.331015:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fe680 x1862022294299520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775768305.331015:0:36:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446631107590089976 : -112966119461640 : ffff99420693ecf8) 00000020:00001000:3.0:1775768305.331020:0:36:0:(cl_object.c:831:cl_env_get()) 1@ffff99420693ecf0 00000020:00000001:3.0:1775768305.331025:0:36:0:(cl_object.c:151:cl_object_top()) ffff99420791a210 -> ffff99420791a850 00000100:00000200:2.0:1775768305.331028:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fe680 x1862022294299520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775768305.331029:0:36:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:3.0:1775768305.331032:0:36:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:3.0:1775768305.331035:0:36:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:3.0:1775768305.331037:0:36:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768305.331041:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00020000:00000001:3.0:1775768305.331041:0:36:0:(lov_io.c:146:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768305.331045:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00020000:00000001:3.0:1775768305.331045:0:36:0:(lov_io.c:183:lov_sub_get()) Process leaving (rc=18446631107612435560 : -112966097116056 : ffff994207e8e468) 00000100:00000001:2.0:1775768305.331050:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:3.0:1775768305.331050:0:36:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000100:00000001:2.0:1775768305.331053:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1775768305.331053:0:36:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768305.331057:0:36:0:(lov_io.c:966:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768305.331061:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fe680 x1862022294299520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775768305.331061:0:36:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768305.331064:0:36:0:(cl_io.c:314:cl_io_lock()) Process entered 00020000:00000001:3.0:1775768305.331067:0:36:0:(lov_io.c:1108:lov_io_lock()) Process entered 00020000:00000001:3.0:1775768305.331070:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000001:2.0:1775768305.331071:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1775768305.331072:0:36:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:3.0:1775768305.331075:0:36:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000100:00000040:2.0:1775768305.331076:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fe680 x1862022294299520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768330 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775768305.331077:0:36:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768305.331082:0:36:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768305.331087:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768305.331092:0:36:0:(lov_io.c:1109:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768305.331093:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:3.0:1775768305.331096:0:36:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:3.0:1775768305.331098:0:36:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768305.331099:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768305.331103:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:3.0:1775768305.331103:0:36:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768305.331106:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1775768305.331107:0:36:0:(cl_io.c:528:cl_io_start()) Process entered 00020000:00000001:3.0:1775768305.331110:0:36:0:(lov_io.c:1114:lov_io_start()) Process entered 00000100:00000001:2.0:1775768305.331111:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00020000:00000001:3.0:1775768305.331112:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775768305.331115:0:36:0:(cl_io.c:528:cl_io_start()) Process entered 00000100:00000001:2.0:1775768305.331116:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00000001:3.0:1775768305.331117:0:36:0:(osc_io.c:1077:osc_io_fsync_start()) Process entered 00000008:00000001:3.0:1775768305.331119:0:36:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000100:00080000:2.0:1775768305.331121:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000008:00000001:3.0:1775768305.331123:0:36:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:3.0:1775768305.331126:0:36:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000100:00080000:2.0:1775768305.331128:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000008:00000001:3.0:1775768305.331128:0:36:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1775768305.331133:0:36:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000100:00000001:2.0:1775768305.331134:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000008:00000001:3.0:1775768305.331135:0:36:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000008:00000001:3.0:1775768305.331138:0:36:0:(osc_cache.c:2295:osc_next_obj()) Process leaving (rc=18446631107754938304 : -112965954613312 : ffff994210674fc0) 00000100:00000040:2.0:1775768305.331141:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fe680 x1862022294299520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768330 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000020:3.0:1775768305.331142:0:36:0:(osc_cache.c:2330:osc_check_rpcs()) obj ffff994210674fc0 ready 0|-|- wr 1|-|+ rd 0|- 9 in flight 00000008:00000001:3.0:1775768305.331149:0:36:0:(osc_cache.c:1797:osc_makes_rpc()) Process entered 00000008:00000020:3.0:1775768305.331152:0:36:0:(osc_cache.c:1818:osc_makes_rpc()) urgent request forcing RPC 00000008:00000001:3.0:1775768305.331179:0:36:0:(osc_cache.c:1819:osc_makes_rpc()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1775768305.331182:0:36:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000008:00000020:3.0:1775768305.331184:0:36:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff994218232380 ready 1|-|+ wr 1|-|+ rd 0|- pageout [0, 18446744073709551615] npages 1: rc=1. 00000008:00000001:3.0:1775768305.331189:0:36:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768305.331190:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294299520:192.168.202.156@tcp:250:kworker.0 00000008:00000001:3.0:1775768305.331194:0:36:0:(osc_io.c:1139:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768305.331197:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1775768305.331198:0:36:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768305.331201:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768305.331204:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fca80 x1862022294299648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00020000:00000001:3.0:1775768305.331204:0:36:0:(lov_io.c:1115:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768305.331208:0:36:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768305.331212:0:36:0:(cl_io.c:555:cl_io_end()) Process entered 00020000:00000001:3.0:1775768305.331214:0:36:0:(lov_io.c:1654:lov_io_fsync_end()) Process entered 00020000:00000001:3.0:1775768305.331217:0:36:0:(lov_io.c:1120:lov_io_end_wrapper()) Process entered 00000020:00000001:3.0:1775768305.331218:0:36:0:(cl_io.c:555:cl_io_end()) Process entered 00000100:00000001:2.0:1775768305.331220:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000001:3.0:1775768305.331221:0:36:0:(cl_io.c:563:cl_io_end()) Process leaving 00000100:00000001:2.0:1775768305.331223:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768305.331223:0:36:0:(lov_io.c:1130:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768305.331227:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00020000:00000001:3.0:1775768305.331227:0:36:0:(lov_io.c:1665:lov_io_fsync_end()) Process leaving 02000000:00000001:2.0:1775768305.331230:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1775768305.331230:0:36:0:(cl_io.c:563:cl_io_end()) Process leaving 00000020:00000001:3.0:1775768305.331233:0:36:0:(cl_io.c:352:cl_io_unlock()) Process entered 02000000:00000001:2.0:1775768305.331234:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768305.331236:0:36:0:(lov_io.c:1208:lov_io_unlock()) Process entered 00000100:00100000:2.0:1775768305.331237:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294299648:192.168.202.156@tcp:38:kworker.0 00020000:00000001:3.0:1775768305.331238:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000100:00000001:2.0:1775768305.331241:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000001:3.0:1775768305.331241:0:36:0:(cl_io.c:352:cl_io_unlock()) Process entered 02000000:00000001:2.0:1775768305.331243:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:3.0:1775768305.331244:0:36:0:(cl_io.c:373:cl_io_unlock()) Process leaving 02000000:00000001:2.0:1775768305.331245:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768305.331246:0:36:0:(lov_io.c:1142:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768305.331248:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00020000:00000001:3.0:1775768305.331249:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768305.331251:0:36:0:(lov_io.c:1211:lov_io_unlock()) Process leaving 00000020:00000001:3.0:1775768305.331252:0:36:0:(cl_io.c:373:cl_io_unlock()) Process leaving 02000000:00000010:2.0:1775768305.331253:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1ec00. 00000020:00000001:3.0:1775768305.331254:0:36:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000020:00000001:3.0:1775768305.331256:0:36:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 02000000:00000001:2.0:1775768305.331257:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768305.331258:0:36:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:3.0:1775768305.331260:0:36:0:(lov_io.c:1189:lov_io_iter_fini()) Process entered 00020000:00000001:3.0:1775768305.331262:0:36:0:(lov_io.c:1086:lov_io_call()) Process entered 00000020:00000001:3.0:1775768305.331263:0:36:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000400:00000200:2.0:1775768305.331264:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:3.0:1775768305.331265:0:36:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000400:00000010:2.0:1775768305.331268:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00020000:00000001:3.0:1775768305.331268:0:36:0:(lov_io.c:1136:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1775768305.331271:0:36:0:(lov_io.c:1103:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775768305.331274:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00020000:00000001:3.0:1775768305.331274:0:36:0:(lov_io.c:1200:lov_io_iter_fini()) Process leaving 00000020:00000001:3.0:1775768305.331277:0:36:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000100:00000200:2.0:1775768305.331279:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294299648, portal 10 00000020:00000001:3.0:1775768305.331279:0:36:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775768305.331282:0:36:0:(cl_io.c:71:cl_io_fini()) Process entered 00000100:00000001:2.0:1775768305.331283:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00020000:00000001:3.0:1775768305.331284:0:36:0:(lov_io.c:675:lov_io_fini()) Process entered 00000100:00000001:2.0:1775768305.331286:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641199232 : -112966068352384 : ffff9942099fca80) 00020000:00000001:3.0:1775768305.331287:0:36:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:3.0:1775768305.331289:0:36:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:3.0:1775768305.331291:0:36:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:3.0:1775768305.331292:0:36:0:(cl_object.c:925:cl_env_put()) 1@ffff99420693ecf0 00000100:00000040:2.0:1775768305.331295:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fca80 x1862022294299648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768330 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1775768305.331296:0:36:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210dcec00. 00000020:00000010:3.0:1775768305.331302:0:36:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207981f60. 00000008:00000010:3.0:1775768305.331306:0:36:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994208b51650. 00000100:00000001:2.0:1775768305.331309:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00020000:00000010:3.0:1775768305.331311:0:36:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994218000000. 00000100:00000040:2.0:1775768305.331314:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00008000:00000010:3.0:1775768305.331316:0:36:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994207e0ade8. 00000400:00000010:2.0:1775768305.331317:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000080:00000010:3.0:1775768305.331321:0:36:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99420541b0a8. 00000100:00000200:2.0:1775768305.331322:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294299648, offset 0 00000080:00000010:3.0:1775768305.331326:0:36:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077c1da8. 00000400:00000200:2.0:1775768305.331329:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768305.331335:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000010:3.0:1775768305.331338:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dce800. 00000400:00000200:2.0:1775768305.331342:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000008:00000010:3.0:1775768305.331343:0:36:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942086f8160. 00000400:00000200:2.0:1775768305.331347:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00020000:00000010:3.0:1775768305.331348:0:36:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994207e8ed00. 00000400:00000200:2.0:1775768305.331352:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00008000:00000010:3.0:1775768305.331353:0:36:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994209270ef0. 00000400:00000200:2.0:1775768305.331356:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000080:00000010:3.0:1775768305.331359:0:36:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942090e3960. 00000400:00000200:2.0:1775768305.331360:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000020:00000010:3.0:1775768305.331365:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcf800. 00000400:00000200:2.0:1775768305.331367:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225514:225514:256:4294967295] 192.168.202.156@tcp LPNI seq info [225510:225510:8:4294967295] 00000020:00000010:3.0:1775768305.331369:0:36:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff99420693ecf0. 00000400:00000200:2.0:1775768305.331374:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00020000:00000001:3.0:1775768305.331374:0:36:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:3.0:1775768305.331376:0:36:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:3.0:1775768305.331380:0:36:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000080:00200000:3.0:1775768305.331384:0:36:0:(vvp_io.c:288:vvp_io_fini()) [0x240000402:0x1017f:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000400:00000200:2.0:1775768305.331386:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000080:00000001:3.0:1775768305.331390:0:36:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:3.0:1775768305.331393:0:36:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000800:00000200:2.0:1775768305.331396:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000020:00001000:3.0:1775768305.331396:0:36:0:(cl_object.c:925:cl_env_put()) 1@ffff99420693e000 00000020:00000010:3.0:1775768305.331400:0:36:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff994210dcf000. 00000800:00000010:2.0:1775768305.331402:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000020:00000010:3.0:1775768305.331403:0:36:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff994207981900. 00000008:00000010:3.0:1775768305.331408:0:36:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff994208b50000. 00000800:00000200:2.0:1775768305.331409:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00020000:00000010:3.0:1775768305.331413:0:36:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff994218000300. 00000800:00000200:2.0:1775768305.331414:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00008000:00000010:3.0:1775768305.331418:0:36:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff994207e08e20. 00000800:00000200:2.1:1775768305.331422:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000080:00000010:3.0:1775768305.331423:0:36:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff99420541b1f8. 00000800:00000200:2.1:1775768305.331426:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000080:00000010:3.0:1775768305.331427:0:36:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff9942077c0248. 00000020:00000010:3.0:1775768305.331432:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcf200. 00000800:00000100:2.0:1775768305.331434:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000008:00000010:3.0:1775768305.331436:0:36:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff9942086f8420. 00000800:00000010:2.0:1775768305.331437:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00020000:00000010:3.0:1775768305.331440:0:36:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff994207e8e400. 00000400:00000200:2.0:1775768305.331442:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00008000:00000010:3.0:1775768305.331444:0:36:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff994209270a90. 00000400:00000200:2.0:1775768305.331447:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000080:00000010:3.0:1775768305.331449:0:36:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff9942090e33c0. 00000400:00000200:2.0:1775768305.331450:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768305.331454:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000020:00000010:3.0:1775768305.331455:0:36:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff994210dcea00. 00000100:00000001:2.0:1775768305.331456:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000020:00000010:3.0:1775768305.331459:0:36:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff99420693e000. 00000100:00000200:2.0:1775768305.331460:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fca80 x1862022294299648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768330 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000080:00000001:3.0:1775768305.331464:0:36:0:(file.c:5284:cl_sync_file_range()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:3.0:1775768305.331468:0:36:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768305.331470:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768305.331472:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fca80 x1862022294299648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768330 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768305.331480:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768305.331481:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768305.331484:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768305.331487:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:2.0:1775768305.331489:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768305.331491:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300230164 00000100:00000001:2.0:1775768305.331494:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300231486 00000100:00000001:2.0:1775768305.331495:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300230164 00000100:00000001:2.0:1775768305.331497:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300231486 00000100:00000001:2.0:1775768305.331498:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768305.331499:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768305.331501:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768305.331503:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768305/real 1775768305] req@ffff9942099fca80 x1862022294299648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768305.331510:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768305.331512:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768305.331513:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768305.331515:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768305.331516:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768305.331518:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768305.331521:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fca80 x1862022294299648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768305.331528:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fca80 x1862022294299648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768305.331533:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768305.331534:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768305.331536:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768305.331537:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768305.331541:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fca80 x1862022294299648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768305.331547:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768305.331551:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fca80 x1862022294299648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768330 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768305.331557:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768305.331559:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768305.331560:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768305.331561:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768305.331562:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768305.331564:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768305.331566:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768305.331568:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768305.331569:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768305.331572:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fca80 x1862022294299648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768330 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768305.331576:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294299648:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768305.331578:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768305.331580:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fce00 x1862022294299776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768305.331584:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768305.331585:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768305.331586:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768305.331587:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768305.331589:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768305.331590:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294299776:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768305.331592:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768305.331593:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768305.331594:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768305.331595:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768305.331598:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1e800. 02000000:00000001:2.0:1775768305.331599:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768305.331601:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768305.331602:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768305.331605:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768305.331606:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294299776, portal 10 00000100:00000001:2.0:1775768305.331608:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768305.331609:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641200128 : -112966068351488 : ffff9942099fce00) 00000100:00000040:2.0:1775768305.331612:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fce00 x1862022294299776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768330 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768305.331617:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768305.331620:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768305.331621:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768305.331623:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294299776, offset 0 00000400:00000200:2.0:1775768305.331627:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768305.331630:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768305.331633:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768305.331636:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768305.331638:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768305.331640:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768305.331642:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768305.331645:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225515:225515:256:4294967295] 192.168.202.156@tcp LPNI seq info [225511:225511:8:4294967295] 00000400:00000200:2.0:1775768305.331649:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768305.331654:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768305.331658:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768305.331661:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768305.331665:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768305.331667:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775768305.331671:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768305.331673:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1775768305.331677:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768305.331679:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768305.331681:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768305.331685:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768305.331687:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768305.331688:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768305.331690:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768305.331693:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fce00 x1862022294299776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768330 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768305.331697:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768305.331699:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fce00 x1862022294299776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768330 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768305.331703:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768305.331704:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768305.331706:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768305.331708:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:2.0:1775768305.331710:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768305.331711:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300230156 00000100:00000001:2.0:1775768305.331712:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300231486 00000100:00000001:2.0:1775768305.331714:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300230156 00000100:00000001:2.0:1775768305.331715:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300231486 00000100:00000001:2.0:1775768305.331716:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768305.331717:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768305.331718:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768305.331721:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768305/real 1775768305] req@ffff9942099fce00 x1862022294299776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768305.331727:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768305.331729:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768305.331730:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768305.331732:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768305.331733:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768305.331735:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768305.331738:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fce00 x1862022294299776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768305.331744:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fce00 x1862022294299776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768305.331749:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768305.331750:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768305.331751:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768305.331753:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768305.331756:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fce00 x1862022294299776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768305.331762:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768305.331765:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fce00 x1862022294299776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768330 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768305.331771:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768305.331772:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768305.331773:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768305.331774:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768305.331776:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768305.331779:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768305.331780:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768305.331783:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768305.331785:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768305.331788:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fce00 x1862022294299776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768330 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768305.331795:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294299776:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768305.331798:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768305.331802:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fdc00 x1862022294299904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768305.331808:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768305.331809:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768305.331811:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768305.331813:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768305.331814:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768305.331816:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294299904:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768305.331819:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768305.331821:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768305.331822:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768305.331824:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768305.331827:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f000. 02000000:00000001:2.0:1775768305.331828:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768305.331832:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768305.331834:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768305.331837:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768305.331839:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294299904, portal 4 00000100:00000001:2.0:1775768305.331842:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768305.331843:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641203712 : -112966068347904 : ffff9942099fdc00) 00000100:00000040:2.0:1775768305.331848:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fdc00 x1862022294299904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768330 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768305.331853:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768305.331855:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768305.331858:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768305.331860:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294299904, offset 0 00000400:00000200:2.0:1775768305.331864:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768305.331867:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768305.331870:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768305.331873:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768305.331875:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768305.331878:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768305.331880:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768305.331884:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225516:225516:256:4294967295] 192.168.202.156@tcp LPNI seq info [225512:225512:8:4294967295] 00000400:00000200:2.0:1775768305.331888:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768305.331894:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768305.331897:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768305.331900:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768305.331903:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768305.331905:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775768305.331909:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768305.331911:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1775768305.331915:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768305.331917:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768305.331919:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768305.331922:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768305.331925:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768305.331927:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768305.331929:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768305.331932:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fdc00 x1862022294299904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768330 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768305.331939:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768305.331953:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fdc00 x1862022294299904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768330 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768305.331959:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768305.331960:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768305.331963:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768305.331966:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:2.0:1775768305.331968:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768305.331970:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300230179 00000100:00000001:2.0:1775768305.331972:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300231486 00000100:00000001:2.0:1775768305.331974:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300230179 00000100:00000001:2.0:1775768305.331976:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300231486 00000100:00000001:2.0:1775768305.331977:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768305.331980:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768305.331981:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768305.331984:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768305/real 1775768305] req@ffff9942099fdc00 x1862022294299904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768305.331991:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768305.331992:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768305.331994:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768305.331996:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768305.331997:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768305.331999:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768305.332002:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fdc00 x1862022294299904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768305.332009:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fdc00 x1862022294299904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768305.332015:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768305.332018:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768305.332020:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768305.332022:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768305.332025:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fdc00 x1862022294299904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768305.332031:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768305.332036:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fdc00 x1862022294299904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768330 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768305.332041:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768305.332044:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768305.332045:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768305.332046:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768305.332047:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768305.332051:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768305.332053:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768305.332057:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768305.332058:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768305.332062:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fdc00 x1862022294299904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768330 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768305.332069:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294299904:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768305.332072:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768305.332076:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fea00 x1862022294300032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768305.332082:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768305.332083:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768305.332085:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768305.332087:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768305.332089:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768305.332091:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294300032:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768305.332094:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768305.332096:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768305.332097:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768305.332098:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768305.332101:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1d000. 02000000:00000001:2.0:1775768305.332103:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768305.332106:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768305.332108:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768305.332111:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768305.332114:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294300032, portal 4 00000100:00000001:2.0:1775768305.332116:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768305.332117:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641207296 : -112966068344320 : ffff9942099fea00) 00000100:00000040:2.0:1775768305.332122:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fea00 x1862022294300032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768330 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768305.332128:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768305.332131:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768305.332132:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768305.332134:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294300032, offset 0 00000400:00000200:2.0:1775768305.332137:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768305.332141:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768305.332145:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768305.332147:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768305.332150:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768305.332152:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768305.332180:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768305.332184:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225517:225517:256:4294967295] 192.168.202.156@tcp LPNI seq info [225513:225513:8:4294967295] 00000400:00000200:2.0:1775768305.332189:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768305.332194:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768305.332198:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768305.332202:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768305.332205:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768305.332208:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775768305.332212:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768305.332215:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1775768305.332218:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768305.332220:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768305.332222:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768305.332225:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768305.332227:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768305.332229:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768305.332231:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768305.332234:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fea00 x1862022294300032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768330 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768305.332241:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768305.332244:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fea00 x1862022294300032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768330 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768305.332250:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768305.332251:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768305.332254:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768305.332257:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:2.0:1775768305.332261:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768305.332263:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300230183 00000100:00000001:2.0:1775768305.332265:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300231486 00000100:00000001:2.0:1775768305.332267:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300230183 00000100:00000001:2.0:1775768305.332269:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300231486 00000100:00000001:2.0:1775768305.332270:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768305.332272:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768305.332274:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768305.332277:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768305/real 1775768305] req@ffff9942099fea00 x1862022294300032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768305.332285:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768305.332286:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768305.332288:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768305.332290:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768305.332291:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768305.332293:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768305.332296:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fea00 x1862022294300032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768305.332304:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fea00 x1862022294300032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768305.332310:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768305.332311:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768305.332313:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768305.332314:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768305.332317:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fea00 x1862022294300032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768305.332323:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768305.332327:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fea00 x1862022294300032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768330 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768305.332333:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768305.332335:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768305.332337:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768305.332338:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768305.332340:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768305.332343:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768305.332346:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768305.332349:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768305.332351:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768305.332354:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fea00 x1862022294300032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768330 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768305.332361:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294300032:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768305.332364:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768305.332366:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768305.332369:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fe680 x1862022294299520/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768330 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768305.332375:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768305.332378:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768305.332381:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1fc00. 02000000:00000001:2.0:1775768305.332384:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768305.332386:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768305.332387:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:2.0:1775768305.332389:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768305.332392:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002b800. 02000000:00000001:2.0:1775768305.332395:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768305.332397:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768305.332399:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fe680. 00000100:00000001:2.0:1775768305.332403:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768305.332404:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768305.332406:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768305.332409:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fca80 x1862022294299648/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768330 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768305.332414:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768305.332415:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768305.332417:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1ec00. 02000000:00000001:2.0:1775768305.332419:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768305.332420:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768305.332421:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:2.0:1775768305.332423:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768305.332424:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028400. 02000000:00000001:2.0:1775768305.332426:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768305.332427:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768305.332428:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fca80. 00000100:00000001:2.0:1775768305.332430:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768305.332431:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768305.332433:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768305.332435:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fce00 x1862022294299776/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768330 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768305.332441:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768305.332442:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768305.332443:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1e800. 02000000:00000001:2.0:1775768305.332445:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768305.332446:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768305.332447:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:2.0:1775768305.332449:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768305.332450:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002bc00. 02000000:00000001:2.0:1775768305.332452:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768305.332453:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768305.332454:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fce00. 00000100:00000001:2.0:1775768305.332456:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768305.332457:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768305.332458:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768305.332461:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fdc00 x1862022294299904/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768330 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768305.332470:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768305.332471:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768305.332472:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f000. 02000000:00000001:2.0:1775768305.332474:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768305.332475:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768305.332475:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:2.0:1775768305.332477:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768305.332478:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029800. 02000000:00000001:2.0:1775768305.332480:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768305.332481:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768305.332483:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fdc00. 00000100:00000001:2.0:1775768305.332484:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768305.332485:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768305.332487:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768305.332489:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fea00 x1862022294300032/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768330 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768305.332494:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768305.332495:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768305.332496:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1d000. 02000000:00000001:2.0:1775768305.332499:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768305.332500:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768305.332501:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:2.0:1775768305.332502:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768305.332503:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029400. 02000000:00000001:2.0:1775768305.332505:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768305.332506:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768305.332507:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fea00. 00000100:00000001:2.0:1775768305.332509:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768305.332510:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768305.332512:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768305.332518:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768305.332520:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768305.332522:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768305.332523:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768305.393265:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768305.393270:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768305.393274:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768305.393275:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768305.393277:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768305.393278:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768305.393280:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768305.393281:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768305.393286:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768305.393287:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768305.393330:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768305.393333:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768305.393335:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768305.457250:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768305.457255:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768305.457256:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768305.457260:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768305.457262:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768305.457265:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768305.457267:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768305.457269:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768305.457271:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768306.353256:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768306.353256:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768306.353256:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768306.353262:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768306.353263:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768306.353263:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768306.353266:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768306.353266:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768306.353267:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768306.353273:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775768306.353274:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775768306.353274:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768306.353278:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768306.353278:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768306.353279:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768306.353282:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768306.353282:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768306.353283:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768306.353287:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768306.353290:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768306.353293:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768306.353294:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768306.353295:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768306.353298:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768306.353299:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768306.353302:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768306.353305:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768306.417258:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768306.417264:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768306.417271:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768306.417272:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768306.417276:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768306.417277:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768306.417280:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768306.417281:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768306.417284:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768306.417285:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768306.417300:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768306.417303:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768306.417306:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768306.481305:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768306.481314:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768306.481316:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768306.481323:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768306.481328:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768306.481333:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768306.481337:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768306.481339:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768306.481343:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768306.673263:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4300231486, at 4300231488 00000100:00000040:0.0:1775768306.673266:0:15036:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4300231486, at 4300231488 00000100:00000010:0.0:1775768306.673283:0:15036:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d6c00. 00000100:00000010:1.0:1775768306.673287:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d6900. 00000100:00000001:3.0:1775768307.377248:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768307.377250:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768307.377256:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768307.377256:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768307.377259:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768307.377259:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768307.377261:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:2.0:1775768307.377265:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775768307.377265:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768307.377266:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768307.377269:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768307.377271:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768307.377271:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1775768307.377272:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:1.0:1775768307.377275:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768307.377275:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768307.377276:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768307.377279:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768307.377281:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768307.377283:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768307.377286:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768307.377287:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768307.377290:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768307.377292:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768307.377292:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768307.377295:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768307.377299:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768307.441257:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768307.441263:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768307.441269:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768307.441270:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768307.441273:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768307.441274:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768307.441276:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768307.441277:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768307.441279:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768307.441280:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768307.441342:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768307.441345:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768307.441348:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768307.505283:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768307.505293:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768307.505295:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768307.505302:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768307.505307:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768307.505312:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768307.505315:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768307.505318:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768307.505321:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768308.401292:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768308.401299:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768308.401303:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768308.401304:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768308.401309:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768308.401311:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768308.401312:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775768308.401317:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775768308.401322:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768308.401323:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768308.401329:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768308.401330:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768308.401334:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768308.401336:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768308.401339:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768308.401340:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768308.401343:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768308.401348:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768308.401355:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768308.401358:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768308.401360:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768308.401364:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775768308.401366:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768308.401369:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768308.401370:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768308.401371:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768308.401373:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768308.465250:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768308.465254:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768308.465259:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768308.465262:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768308.465264:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768308.465266:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768308.465268:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768308.465270:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768308.465271:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768308.465272:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768308.465275:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768308.465276:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768308.465276:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768308.465277:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768308.465278:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768308.465279:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768308.465280:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768308.465281:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768308.465282:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768308.465284:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768308.465284:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768308.465285:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768308.465286:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768308.465286:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768308.465287:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768308.465288:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768308.465290:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768308.465291:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768308.465292:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768308.465292:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768308.465293:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768308.465294:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768308.465295:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768308.465296:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768308.465340:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768308.465342:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768308.465343:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768308.529204:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768308.529208:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768308.529210:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768308.529214:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768308.529216:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768308.529219:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768308.529221:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768308.529222:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768308.529224:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768309.425258:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768309.425261:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768309.425263:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768309.425265:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768309.425265:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768309.425267:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768309.425270:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:0.0:1775768309.425271:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775768309.425276:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768309.425276:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768309.425280:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768309.425281:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768309.425286:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768309.425288:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768309.425290:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768309.425290:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768309.425292:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768309.425295:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768309.425363:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768309.425365:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768309.425366:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768309.425368:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768309.425369:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768309.425371:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768309.425372:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768309.425373:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768309.425374:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768309.489245:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768309.489249:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768309.489253:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768309.489254:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768309.489256:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768309.489257:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768309.489259:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768309.489260:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768309.489262:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768309.489263:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768309.489304:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768309.489306:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768309.489309:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768309.553255:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768309.553260:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768309.553261:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768309.553265:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768309.553268:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768309.553271:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768309.553273:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768309.553274:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768309.553277:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768310.449251:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768310.449251:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768310.449259:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768310.449259:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768310.449262:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768310.449262:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768310.449263:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768310.449264:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775768310.449265:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768310.449269:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768310.449270:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:2.0:1775768310.449271:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775768310.449271:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00100000:1.0:1775768310.449272:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775768310.449274:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775768310.449276:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768310.449278:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768310.449278:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1775768310.449279:0:8:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768310.449280:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768310.449283:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768310.449284:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768310.449284:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768310.449285:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775768310.449287:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768310.449290:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768310.449291:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1775768310.449293:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5562 00000100:00000001:2.0:1775768310.449295:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768310.449298:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768310.449299:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000400:00000200:0.0:1775768310.449303:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:3.0:1775768310.449303:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768310.449309:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768310.449312:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768310.449315:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768310.449317:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768310.449320:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768310.449322:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768310.449324:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768310.449327:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768310.449329:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:0.0:1775768310.449330:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768310.449332:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:0.0:1775768310.449334:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:0.0:1775768310.449336:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:0.0:1775768310.449338:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768310.449340:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768310.449342:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768310.449343:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768310.449345:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:0.0:1775768310.449348:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768310.449349:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768310.449351:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768310.449354:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768310.449356:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768310.449357:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775768310.449359:0:8:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768310.449366:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fc380. 00000020:00000040:0.0:1775768310.449369:0:8:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:0.0:1775768310.449371:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768310.449372:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768310.449374:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768310.449376:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768310.449379:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768310.449384:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220028000. 00000100:00000001:0.0:1775768310.449389:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768310.449395:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fc380 x1862022294300160/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768310.449402:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768310.449405:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768310.449408:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002957419c] to pc [ptlrpcd_rcv+-1] req@ffff9942099fc380 x1862022294300160/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768310.449421:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768310.449423:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768310.449425:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775768310.449426:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768310.449427:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768310.449429:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768310.449431:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5562 00000400:00000200:0.0:1775768310.449433:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768310.449435:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768310.449437:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768310.449439:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768310.449440:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775768310.449440:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1775768310.449443:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768310.449443:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768310.449446:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:2.0:1775768310.449446:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000001:0.0:1775768310.449449:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768310.449449:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:0.0:1775768310.449451:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768310.449453:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000100:00000040:2.0:1775768310.449453:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fc380 x1862022294300160/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775768310.449457:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:0.0:1775768310.449461:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000040:0.0:1775768310.449467:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768310.449469:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775768310.449469:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768310.449472:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768310.449473:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768310.449476:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 02000000:00000001:2.0:1775768310.449476:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:0.0:1775768310.449478:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 02000000:00000001:2.0:1775768310.449479:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:0.0:1775768310.449483:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 02000000:00000001:2.0:1775768310.449483:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768310.449486:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768310.449489:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:2.0:1775768310.449489:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294300160:192.168.202.156@tcp:250:kworker.0 00000100:00000001:2.0:1775768310.449494:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768310.449495:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768310.449497:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768310.449497:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768310.449499:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768310.449500:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:2.0:1775768310.449503:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:0.0:1775768310.449504:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775768310.449507:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775768310.449508:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1d000. 00000100:00000001:0.0:1775768310.449509:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768310.449511:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768310.449515:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fed80. 00000400:00000200:2.0:1775768310.449518:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000040:0.0:1775768310.449520:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000400:00000010:2.0:1775768310.449522:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 02000000:00000001:0.0:1775768310.449524:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768310.449527:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775768310.449527:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000001:0.0:1775768310.449530:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768310.449533:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768310.449535:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768310.449540:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002a800. 00000100:00000200:2.0:1775768310.449540:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294300160, portal 25 00000100:00000001:2.0:1775768310.449544:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768310.449546:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641197440 : -112966068354176 : ffff9942099fc380) 00000100:00000001:0.0:1775768310.449547:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768310.449551:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fc380 x1862022294300160/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768335 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768310.449555:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fed80 x1862022294300288/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775768310.449560:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768310.449562:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768310.449565:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768310.449569:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294300160, offset 0 00000100:00000001:0.0:1775768310.449575:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:2.0:1775768310.449578:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00000001:0.0:1775768310.449579:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768310.449583:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000c09b4d8] to pc [ptlrpcd_rcv+-1] req@ffff9942099fed80 x1862022294300288/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1775768310.449587:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768310.449593:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00000001:0.0:1775768310.449594:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768310.449597:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:2.0:1775768310.449598:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:0.0:1775768310.449602:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000200:2.0:1775768310.449603:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000100:00000001:0.0:1775768310.449604:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768310.449607:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:2.0:1775768310.449609:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1775768310.449610:0:8:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:2.0:1775768310.449613:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00080000:0.0:1775768310.449615:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5562 00000400:00000200:0.0:1775768310.449621:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768310.449622:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225518:225518:256:4294967295] 192.168.202.156@tcp LPNI seq info [225514:225514:8:4294967295] 00000400:00000200:0.0:1775768310.449625:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768310.449629:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768310.449631:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000200:2.0:1775768310.449633:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768310.449634:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768310.449637:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768310.449638:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768310.449640:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768310.449641:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:0.0:1775768310.449642:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768310.449643:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:0.0:1775768310.449646:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000020:00000001:0.0:1775768310.449650:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000400:00000200:2.0:1775768310.449651:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000100:00000040:0.0:1775768310.449654:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768310.449657:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768310.449659:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768310.449661:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768310.449663:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000800:00000200:2.0:1775768310.449665:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00080000:0.0:1775768310.449667:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768310.449671:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768310.449673:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000800:00000010:2.0:1775768310.449673:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 02000000:00000001:0.0:1775768310.449676:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768310.449679:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768310.449683:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000800:00000200:2.0:1775768310.449683:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000008:00100000:0.0:1775768310.449687:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000200:2.0:1775768310.449688:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000008:00000001:0.0:1775768310.449693:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775768310.449696:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:0.0:1775768310.449697:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775768310.449699:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1775768310.449703:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768310.449705:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768310.449709:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000010:0.0:1775768310.449710:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fd180. 00000400:00000200:2.0:1775768310.449714:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000020:00000040:0.0:1775768310.449715:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000400:00000200:2.0:1775768310.449718:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 02000000:00000001:0.0:1775768310.449719:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768310.449721:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775768310.449721:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:0.0:1775768310.449725:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768310.449725:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1775768310.449726:0:2399:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:0.0:1775768310.449727:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1775768310.449731:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fc380 x1862022294300160/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768335 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1775768310.449734:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220029400. 00000100:00000001:0.0:1775768310.449739:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768310.449741:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00100000:0.0:1775768310.449745:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fd180 x1862022294300416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:2.0:1775768310.449746:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fc380 x1862022294300160/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768335 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768310.449761:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768310.449762:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768310.449766:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768310.449768:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:2.0:1775768310.449771:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768310.449773:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768310.449774:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4300236605 00000100:00000010:2.0:1775768310.449780:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d6360. 00000100:00000040:0.0:1775768310.449781:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004ec16d17] to pc [ptlrpcd_rcv+-1] req@ffff9942099fd180 x1862022294300416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768310.449785:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1775768310.449792:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4300236605 00000100:00000001:0.0:1775768310.449794:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775768310.449794:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d6660. 00000100:00080000:0.0:1775768310.449800:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775768310.449800:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1775768310.449803:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:0.0:1775768310.449806:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1775768310.449806:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768310.449810:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1775768310.449810:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00080000:0.0:1775768310.449813:0:8:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768310.449816:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00100000:2.0:1775768310.449816:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768310/real 1775768310] req@ffff9942099fc380 x1862022294300160/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768310.449820:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5562 00000400:00000200:0.0:1775768310.449828:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768310.449833:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768310.449836:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768310.449838:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000001:2.0:1775768310.449838:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1775768310.449840:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000001:2.0:1775768310.449841:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000040:0.0:1775768310.449843:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000010:2.0:1775768310.449845:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000100:00000001:0.0:1775768310.449846:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768310.449850:0:8:0:(genops.c:811:class_conn2export()) Process entered 00000400:00000200:2.0:1775768310.449850:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000020:00000040:0.0:1775768310.449853:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000400:00000010:2.0:1775768310.449854:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000020:00000001:0.0:1775768310.449857:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:2.0:1775768310.449858:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000040:0.0:1775768310.449860:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000100:00000200:2.0:1775768310.449864:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fc380 x1862022294300160/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775768310.449865:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000020:00000001:0.0:1775768310.449871:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:0.0:1775768310.449880:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000200:2.0:1775768310.449880:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fc380 x1862022294300160/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768310.449885:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775768310.449889:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000040:0.0:1775768310.449904:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768310.449904:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768310.449908:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775768310.449909:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:2.0:1775768310.449910:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:0.0:1775768310.449912:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00100000:2.0:1775768310.449915:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fc380 x1862022294300160/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768310.449918:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768310.449925:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768310.449930:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1775768310.449930:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1775768310.449934:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1775768310.449935:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fc380 x1862022294300160/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768335 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1775768310.449940:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768310.449944:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775768310.449948:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000100:00000001:2.0:1775768310.449952:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00000001:0.0:1775768310.449953:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768310.449955:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768310.449957:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768310.449957:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768310.449959:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768310.449962:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768310.449963:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000010:0.0:1775768310.449964:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ff480. 00000100:00080000:2.0:1775768310.449965:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000020:00000040:0.0:1775768310.449968:0:8:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00000100:00080000:2.0:1775768310.449970:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 02000000:00000001:0.0:1775768310.449972:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768310.449974:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768310.449974:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:0.0:1775768310.449979:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768310.449980:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768310.449983:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768310.449983:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fc380 x1862022294300160/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768335 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1775768310.449987:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220029800. 00000100:00000001:0.0:1775768310.449991:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768310.449999:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ff480 x1862022294300544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768310.450000:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294300160:192.168.202.156@tcp:250:kworker.0 00000100:00000001:2.0:1775768310.450010:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768310.450011:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768310.450015:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768310.450015:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775768310.450022:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e73dcd12] to pc [ptlrpcd_rcv+-1] req@ffff9942099ff480 x1862022294300544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775768310.450023:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fc380 x1862022294300160/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768335 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768310.450042:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768310.450044:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 00000100:00080000:0.0:1775768310.450045:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:2.0:1775768310.450047:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000100:00080000:0.0:1775768310.450050:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 02000000:00000010:2.0:1775768310.450050:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1d000. 00000100:00000001:0.0:1775768310.450051:0:8:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768310.450054:0:8:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768310.450056:0:8:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768310.450058:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5562 00000400:00000200:0.0:1775768310.450060:0:8:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768310.450063:0:8:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768310.450064:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 02000000:00000001:2.0:1775768310.450064:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768310.450066:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000040:0.0:1775768310.450069:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000040:2.0:1775768310.450069:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000100:00000001:0.0:1775768310.450073:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:2.0:1775768310.450073:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768310.450076:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028000. 00000100:00000040:0.0:1775768310.450077:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768310.450081:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768310.450083:0:8:0:(genops.c:811:class_conn2export()) Process entered 02000000:00000001:2.0:1775768310.450083:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000020:00000040:0.0:1775768310.450086:0:8:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 02000000:00000001:2.0:1775768310.450086:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000020:00000001:0.0:1775768310.450087:0:8:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000010:2.0:1775768310.450089:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fc380. 00000020:00000040:0.0:1775768310.450090:0:8:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:0.0:1775768310.450093:0:8:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:0.0:1775768310.450095:0:8:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000001:2.0:1775768310.450095:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768310.450098:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768310.450099:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768310.450103:0:8:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768310.450104:0:8:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768310.450104:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768310.450107:0:8:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768310.450110:0:8:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00000001:2.0:1775768310.450114:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768310.450116:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768310.450121:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775768310.450121:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775768310.450125:0:8:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768310.450125:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1775768310.450127:0:8:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1775768310.450128:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1775768310.450130:0:8:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768310.450133:0:8:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768310.450133:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fed80 x1862022294300288/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768310.450137:0:8:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775768310.450140:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:0.0:1775768310.450144:0:8:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768310.450146:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768310.450147:0:8:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768310.450149:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768310.450150:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fd880. 02000000:00000001:2.0:1775768310.450151:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768310.450154:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768310.450171:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1775768310.450174:0:8:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 00000100:00100000:2.0:1775768310.450174:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294300288:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768310.450179:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768310.450180:0:8:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775768310.450181:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775768310.450182:0:8:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768310.450183:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768310.450184:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775768310.450185:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1775768310.450186:0:8:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768310.450189:0:8:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775768310.450189:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1d000. 02000000:00000001:2.0:1775768310.450191:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768310.450192:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002bc00. 00000400:00000200:2.0:1775768310.450196:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:0.0:1775768310.450197:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775768310.450199:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000100:00100000:0.0:1775768310.450203:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fd880 x1862022294300672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:2.0:1775768310.450204:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768310.450208:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294300288, portal 10 00000100:00000001:2.0:1775768310.450213:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768310.450215:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641208192 : -112966068343424 : ffff9942099fed80) 00000100:00000001:0.0:1775768310.450222:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768310.450224:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768310.450226:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fed80 x1862022294300288/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768335 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775768310.450229:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007f8a5f3b] to pc [ptlrpcd_rcv+-1] req@ffff9942099fd880 x1862022294300672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768310.450251:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1775768310.450252:0:8:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768310.450256:0:8:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5572) 00000100:00000040:2.0:1775768310.450257:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768310.450261:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768310.450264:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294300288, offset 0 00000400:00000200:2.0:1775768310.450269:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768310.450274:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768310.450278:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768310.450281:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768310.450284:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768310.450288:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768310.450290:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768310.450294:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225519:225519:256:4294967295] 192.168.202.156@tcp LPNI seq info [225515:225515:8:4294967295] 00000400:00000200:2.0:1775768310.450299:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768310.450308:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768310.450314:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768310.450316:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768310.450321:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768310.450322:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775768310.450327:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768310.450329:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1775768310.450332:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768310.450333:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768310.450335:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768310.450339:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768310.450340:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768310.450341:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768310.450343:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768310.450346:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fed80 x1862022294300288/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768335 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768310.450351:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768310.450352:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fed80 x1862022294300288/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768335 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768310.450357:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768310.450357:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768310.450359:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768310.450361:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:2.0:1775768310.450363:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768310.450365:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300235283 00000100:00000001:2.0:1775768310.450368:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300236605 00000100:00000001:2.0:1775768310.450370:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300235283 00000100:00000001:2.0:1775768310.450371:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300236605 00000100:00000001:2.0:1775768310.450374:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768310.450376:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768310.450377:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768310.450380:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768310/real 1775768310] req@ffff9942099fed80 x1862022294300288/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768310.450388:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768310.450389:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768310.450390:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768310.450393:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768310.450394:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768310.450396:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768310.450399:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fed80 x1862022294300288/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768310.450408:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fed80 x1862022294300288/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768310.450414:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768310.450415:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768310.450416:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768310.450417:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768310.450419:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fed80 x1862022294300288/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768310.450423:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768310.450426:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fed80 x1862022294300288/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768335 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768310.450429:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768310.450431:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768310.450432:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768310.450433:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768310.450434:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768310.450436:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768310.450437:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768310.450439:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768310.450440:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768310.450442:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fed80 x1862022294300288/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768335 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768310.450447:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294300288:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768310.450449:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768310.450451:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fd180 x1862022294300416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768310.450454:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768310.450455:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768310.450456:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768310.450457:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768310.450459:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768310.450460:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294300416:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768310.450462:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768310.450463:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768310.450464:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768310.450465:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768310.450467:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f000. 02000000:00000001:2.0:1775768310.450468:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768310.450471:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768310.450472:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768310.450474:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768310.450476:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294300416, portal 10 00000100:00000001:2.0:1775768310.450477:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768310.450478:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641201024 : -112966068350592 : ffff9942099fd180) 00000100:00000040:2.0:1775768310.450481:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fd180 x1862022294300416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768335 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768310.450485:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768310.450486:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768310.450487:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768310.450489:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294300416, offset 0 00000400:00000200:2.0:1775768310.450491:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768310.450493:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768310.450495:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768310.450497:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768310.450498:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768310.450500:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768310.450501:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768310.450503:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225520:225520:256:4294967295] 192.168.202.156@tcp LPNI seq info [225516:225516:8:4294967295] 00000400:00000200:2.0:1775768310.450506:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768310.450510:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768310.450512:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768310.450514:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768310.450517:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768310.450518:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775768310.450520:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768310.450522:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1775768310.450524:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768310.450525:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768310.450527:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768310.450529:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768310.450530:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768310.450531:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768310.450533:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768310.450535:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fd180 x1862022294300416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768335 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768310.450542:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768310.450544:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fd180 x1862022294300416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768335 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768310.450547:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768310.450548:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768310.450550:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768310.450551:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:2.0:1775768310.450553:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768310.450554:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300235275 00000100:00000001:2.0:1775768310.450555:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300236605 00000100:00000001:2.0:1775768310.450556:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300235275 00000100:00000001:2.0:1775768310.450558:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300236605 00000100:00000001:2.0:1775768310.450559:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768310.450560:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768310.450561:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768310.450563:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768310/real 1775768310] req@ffff9942099fd180 x1862022294300416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768310.450567:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768310.450567:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768310.450568:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768310.450569:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768310.450570:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768310.450571:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768310.450573:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fd180 x1862022294300416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768310.450577:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fd180 x1862022294300416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768310.450581:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768310.450582:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768310.450583:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768310.450584:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768310.450585:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fd180 x1862022294300416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768310.450589:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768310.450591:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fd180 x1862022294300416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768335 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768310.450594:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768310.450595:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768310.450596:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768310.450597:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768310.450598:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768310.450599:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768310.450600:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768310.450602:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768310.450603:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768310.450605:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fd180 x1862022294300416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768335 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768310.450609:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294300416:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768310.450611:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768310.450612:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ff480 x1862022294300544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768310.450616:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768310.450617:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768310.450618:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768310.450619:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768310.450620:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768310.450622:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294300544:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768310.450624:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768310.450625:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768310.450626:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768310.450627:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768310.450628:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1e800. 02000000:00000001:2.0:1775768310.450629:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768310.450631:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768310.450632:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768310.450634:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768310.450636:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294300544, portal 4 00000100:00000001:2.0:1775768310.450636:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768310.450637:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641209984 : -112966068341632 : ffff9942099ff480) 00000100:00000040:2.0:1775768310.450640:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ff480 x1862022294300544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768335 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768310.450643:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768310.450645:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768310.450646:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768310.450647:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294300544, offset 0 00000400:00000200:2.0:1775768310.450649:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768310.450650:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768310.450652:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768310.450653:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768310.450655:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768310.450656:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768310.450657:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768310.450659:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225521:225521:256:4294967295] 192.168.202.156@tcp LPNI seq info [225517:225517:8:4294967295] 00000400:00000200:2.0:1775768310.450662:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768310.450665:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768310.450667:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768310.450668:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768310.450670:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768310.450672:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775768310.450674:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768310.450676:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1775768310.450678:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768310.450679:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768310.450680:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768310.450682:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768310.450683:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768310.450684:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768310.450686:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768310.450687:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ff480 x1862022294300544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768335 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768310.450692:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768310.450693:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ff480 x1862022294300544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768335 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768310.450697:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768310.450697:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768310.450699:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768310.450700:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:2.0:1775768310.450702:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768310.450703:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300235298 00000100:00000001:2.0:1775768310.450704:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300236605 00000100:00000001:2.0:1775768310.450705:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300235298 00000100:00000001:2.0:1775768310.450707:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300236605 00000100:00000001:2.0:1775768310.450709:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768310.450711:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768310.450713:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768310.450715:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768310/real 1775768310] req@ffff9942099ff480 x1862022294300544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768310.450722:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768310.450724:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768310.450725:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768310.450726:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768310.450728:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768310.450730:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768310.450732:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ff480 x1862022294300544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768310.450739:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ff480 x1862022294300544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768310.450746:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768310.450747:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768310.450748:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768310.450749:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768310.450753:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ff480 x1862022294300544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768310.450759:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768310.450762:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ff480 x1862022294300544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768335 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768310.450768:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768310.450770:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768310.450771:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768310.450772:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768310.450774:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768310.450777:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768310.450779:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768310.450782:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768310.450784:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768310.450787:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ff480 x1862022294300544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768335 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768310.450797:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294300544:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768310.450799:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768310.450800:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768310.450802:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fed80 x1862022294300288/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768335 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768310.450805:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768310.450806:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768310.450807:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1d000. 02000000:00000001:2.0:1775768310.450809:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768310.450810:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768310.450812:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:2.0:1775768310.450814:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768310.450815:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002a800. 02000000:00000001:2.0:1775768310.450817:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768310.450818:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768310.450819:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fed80. 00000100:00000001:2.0:1775768310.450822:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768310.450823:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768310.450825:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768310.450827:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fd180 x1862022294300416/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768335 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768310.450833:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768310.450834:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768310.450836:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f000. 02000000:00000001:2.0:1775768310.450837:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768310.450839:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768310.450840:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:2.0:1775768310.450841:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768310.450843:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029400. 02000000:00000001:2.0:1775768310.450845:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768310.450846:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768310.450847:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fd180. 00000100:00000001:2.0:1775768310.450850:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768310.450851:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768310.450852:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768310.450855:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ff480 x1862022294300544/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768335 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768310.450861:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768310.450862:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768310.450863:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1e800. 02000000:00000001:2.0:1775768310.450866:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768310.450867:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768310.450868:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:2.0:1775768310.450869:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768310.450870:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029800. 02000000:00000001:2.0:1775768310.450871:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768310.450872:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768310.450872:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ff480. 00000100:00000001:2.0:1775768310.450873:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768310.450874:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768310.450875:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768310.450877:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768310.450878:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768310.450879:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768310.450880:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768310.450881:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768310.450883:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fd880 x1862022294300672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768310.450889:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768310.450902:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768310.450905:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768310.450906:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768310.450908:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768310.450911:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294300672:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768310.450914:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768310.450916:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768310.450917:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768310.450919:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768310.450922:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1e800. 02000000:00000001:2.0:1775768310.450924:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768310.450927:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768310.450929:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768310.450932:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768310.450935:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294300672, portal 4 00000100:00000001:2.0:1775768310.450937:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768310.450938:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641202816 : -112966068348800 : ffff9942099fd880) 00000100:00000040:2.0:1775768310.450944:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fd880 x1862022294300672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768335 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768310.450950:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768310.450952:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768310.450954:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768310.450956:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294300672, offset 0 00000400:00000200:2.0:1775768310.450958:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768310.450961:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768310.450964:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768310.450966:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768310.450967:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768310.450969:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768310.450970:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768310.450972:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225522:225522:256:4294967295] 192.168.202.156@tcp LPNI seq info [225518:225518:8:4294967295] 00000400:00000200:2.0:1775768310.450974:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768310.450978:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768310.450981:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768310.450983:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768310.450985:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768310.450987:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775768310.450989:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768310.450991:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1775768310.450993:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768310.450994:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768310.450997:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768310.450999:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768310.451000:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768310.451001:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768310.451003:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768310.451005:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fd880 x1862022294300672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768335 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768310.451009:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768310.451011:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fd880 x1862022294300672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768335 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768310.451014:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768310.451015:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768310.451017:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768310.451019:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:2.0:1775768310.451020:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768310.451021:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300235301 00000100:00000001:2.0:1775768310.451023:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300236605 00000100:00000001:2.0:1775768310.451024:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300235301 00000100:00000001:2.0:1775768310.451026:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300236605 00000100:00000001:2.0:1775768310.451027:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768310.451028:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768310.451029:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768310.451032:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768310/real 1775768310] req@ffff9942099fd880 x1862022294300672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768310.451039:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768310.451040:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768310.451041:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768310.451043:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768310.451044:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768310.451046:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768310.451048:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fd880 x1862022294300672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768310.451056:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fd880 x1862022294300672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768310.451061:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768310.451063:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768310.451064:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768310.451066:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768310.451068:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fd880 x1862022294300672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768310.451072:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768310.451076:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fd880 x1862022294300672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768335 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768310.451082:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768310.451084:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768310.451086:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768310.451087:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768310.451088:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768310.451091:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768310.451093:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768310.451095:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768310.451097:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768310.451099:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fd880 x1862022294300672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768335 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768310.451104:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294300672:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768310.451106:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768310.451107:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768310.451108:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fd880 x1862022294300672/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768335 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768310.451112:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768310.451113:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768310.451114:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1e800. 02000000:00000001:2.0:1775768310.451115:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768310.451116:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768310.451117:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:2.0:1775768310.451118:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768310.451119:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002bc00. 02000000:00000001:2.0:1775768310.451120:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768310.451121:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768310.451122:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fd880. 00000100:00000001:2.0:1775768310.451123:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768310.451124:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768310.451125:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768310.451127:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768310.451128:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768310.451129:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768310.451130:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768310.513251:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768310.513255:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768310.513259:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768310.513260:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768310.513263:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768310.513264:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768310.513266:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768310.513267:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768310.513269:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768310.513270:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768310.513285:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768310.513286:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768310.513288:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768310.577258:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768310.577265:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768310.577266:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768310.577270:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768310.577272:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768310.577276:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768310.577277:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768310.577279:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768310.577281:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768311.473226:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768311.473228:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768311.473228:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768311.473233:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768311.473234:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768311.473234:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768311.473235:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768311.473238:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768311.473238:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768311.473240:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775768311.473243:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768311.473245:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1775768311.473245:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775768311.473246:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768311.473249:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768311.473249:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768311.473250:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768311.473255:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768311.473255:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768311.473258:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768311.473260:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768311.473261:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768311.473268:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768311.473270:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768311.473271:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768311.473274:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768311.473277:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768311.537234:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768311.537239:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768311.537241:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768311.537244:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768311.537247:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768311.537249:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768311.537250:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768311.537252:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768311.537254:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768311.537255:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768311.537257:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768311.537258:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768311.537259:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768311.537260:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768311.537263:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768311.537264:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768311.537314:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768311.537317:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768311.537319:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768311.601207:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768311.601213:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768311.601214:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768311.601218:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768311.601220:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768311.601223:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768311.601225:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768311.601227:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768311.601229:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768311.793241:0:15036:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4300236605, at 4300236608 00000100:00000040:1.0:1775768311.793241:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4300236605, at 4300236608 00000100:00000010:1.0:1775768311.793253:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d6660. 00000100:00000010:0.0:1775768311.793255:0:15036:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d6360. 00000100:00000001:1.0:1775768312.497248:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768312.497248:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768312.497248:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768312.497253:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768312.497255:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768312.497255:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768312.497255:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768312.497258:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768312.497258:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768312.497260:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775768312.497262:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775768312.497262:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768312.497264:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768312.497266:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768312.497266:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768312.497267:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768312.497269:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768312.497269:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768312.497270:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768312.497273:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768312.497275:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768312.497279:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768312.497281:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768312.497284:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768312.497284:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768312.497285:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768312.497287:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768312.561249:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768312.561253:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768312.561258:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768312.561259:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768312.561262:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768312.561263:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768312.561266:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768312.561267:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768312.561269:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768312.561270:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768312.561298:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768312.561300:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768312.561301:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768312.625236:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768312.625241:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768312.625242:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768312.625246:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768312.625249:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768312.625252:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768312.625254:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768312.625256:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768312.625257:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768313.521220:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768313.521226:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768313.521227:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768313.521228:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768313.521228:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:3.0:1775768313.521232:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768313.521233:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768313.521233:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768313.521235:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768313.521235:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768313.521236:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:2.0:1775768313.521239:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:1.0:1775768313.521240:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768313.521240:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768313.521244:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768313.521244:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768313.521244:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768313.521245:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768313.521248:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768313.521248:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768313.521248:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768313.521253:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768313.521256:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768313.521258:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768313.521258:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768313.521260:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768313.521262:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768313.585252:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768313.585256:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768313.585260:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768313.585261:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768313.585264:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768313.585265:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768313.585267:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768313.585268:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768313.585270:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768313.585271:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768313.585341:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768313.585345:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768313.585348:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768313.649253:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768313.649258:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768313.649259:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768313.649263:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768313.649266:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768313.649269:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768313.649270:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768313.649272:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768313.649274:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768314.545257:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768314.545271:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768314.545274:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768314.545278:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00100000:3.0:1775768314.545282:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768314.545284:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768314.545287:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768314.545290:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1775768314.545295:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:3.0:1775768314.545297:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768314.545304:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768314.545308:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768314.545314:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768314.545318:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768314.545412:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768314.545416:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768314.545418:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768314.545421:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768314.545424:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768314.545427:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768314.545448:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768314.545449:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768314.545454:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768314.545458:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768314.545730:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768314.545733:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768314.545736:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768314.609264:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768314.609270:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768314.609275:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768314.609276:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768314.609279:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768314.609280:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768314.609282:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768314.609283:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768314.609286:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768314.609287:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768314.609304:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768314.609306:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768314.609309:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768314.673257:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768314.673261:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768314.673262:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768314.673266:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768314.673269:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768314.673272:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768314.673273:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768314.673275:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768314.673277:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768315.569230:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768315.569237:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768315.569239:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775768315.569245:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768315.569248:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768315.569253:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768315.569256:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768315.569260:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768315.569263:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768315.569275:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768315.569282:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775768315.569286:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768315.569291:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768315.569297:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775768315.569301:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:0.0:1775768315.569306:0:13471:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5567 00000100:00000001:2.0:1775768315.569307:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768315.569310:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768315.569314:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000400:00000200:0.0:1775768315.569319:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768315.569328:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768315.569336:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768315.569340:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000400:00000200:0.0:1775768315.569343:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:2.0:1775768315.569344:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1775768315.569348:0:13471:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1775768315.569350:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768315.569353:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768315.569356:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768315.569359:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768315.569361:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768315.569365:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768315.569367:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:0.0:1775768315.569370:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768315.569374:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:0.0:1775768315.569377:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:0.0:1775768315.569380:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000001:1.0:1775768315.569381:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000040:0.0:1775768315.569387:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768315.569389:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768315.569392:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768315.569393:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768315.569398:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00100000:1.0:1775768315.569398:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775768315.569405:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:1.0:1775768315.569406:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000040:0.0:1775768315.569411:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00000001:1.0:1775768315.569411:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768315.569417:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1775768315.569421:0:13471:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775768315.569423:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768315.569427:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768315.569429:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768315.569431:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768315.569436:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768315.569439:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768315.569443:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775768315.569446:0:13471:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768315.569455:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fe300. 00000020:00000040:0.0:1775768315.569458:0:13471:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:0.0:1775768315.569462:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768315.569463:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768315.569465:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768315.569468:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768315.569473:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768315.569480:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220028400. 00000100:00000001:0.0:1775768315.569487:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768315.569496:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fe300 x1862022294300800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775768315.569505:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768315.569511:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768315.569516:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008a1e3639] to pc [ptlrpcd_rcv+-1] req@ffff9942099fe300 x1862022294300800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768315.569540:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768315.569544:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768315.569548:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775768315.569549:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768315.569551:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768315.569554:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768315.569557:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5567 00000400:00000200:0.0:1775768315.569562:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:0.0:1775768315.569566:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:0.0:1775768315.569569:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1775768315.569570:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1775768315.569572:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768315.569576:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775768315.569576:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1775768315.569580:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768315.569580:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1775768315.569585:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000040:2.0:1775768315.569589:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fe300 x1862022294300800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775768315.569590:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768315.569592:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000020:00000001:0.0:1775768315.569596:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768315.569600:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 00000020:00000001:0.0:1775768315.569604:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000001:2.0:1775768315.569606:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000001:0.0:1775768315.569608:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000100:00000001:2.0:1775768315.569609:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768315.569615:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 02000000:00000001:2.0:1775768315.569616:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1775768315.569618:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768315.569620:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 02000000:00000001:2.0:1775768315.569620:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768315.569624:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768315.569627:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 02000000:00000001:2.0:1775768315.569627:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768315.569630:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768315.569632:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768315.569634:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:2.0:1775768315.569635:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294300800:192.168.202.156@tcp:250:kworker.0 02000000:00000001:0.0:1775768315.569640:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768315.569641:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775768315.569644:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768315.569646:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:2.0:1775768315.569648:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000008:00100000:0.0:1775768315.569650:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1775768315.569652:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1775768315.569653:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768315.569657:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768315.569659:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000010:0.0:1775768315.569660:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fc700. 00000020:00000040:0.0:1775768315.569662:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:0.0:1775768315.569664:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768315.569664:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768315.569666:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768315.569667:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000010:2.0:1775768315.569667:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1e800. 02000000:00000001:0.0:1775768315.569669:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768315.569671:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768315.569674:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002b800. 00000100:00000001:0.0:1775768315.569680:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768315.569680:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768315.569686:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000100:00100000:0.0:1775768315.569688:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fc700 x1862022294300928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:2.0:1775768315.569696:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000001:0.0:1775768315.569697:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768315.569702:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768315.569706:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003283b20a] to pc [ptlrpcd_rcv+-1] req@ffff9942099fc700 x1862022294300928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768315.569707:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294300800, portal 25 00000100:00000001:2.0:1775768315.569712:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768315.569715:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641205504 : -112966068346112 : ffff9942099fe300) 00000100:00000001:0.0:1775768315.569723:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768315.569727:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fe300 x1862022294300800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768340 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768315.569728:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768315.569735:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775768315.569737:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768315.569739:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768315.569742:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768315.569748:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5567 00000100:00000001:2.0:1775768315.569752:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:0.0:1775768315.569756:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000040:2.0:1775768315.569760:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000200:0.0:1775768315.569763:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000400:00000010:2.0:1775768315.569764:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00080000:0.0:1775768315.569767:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000200:2.0:1775768315.569770:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294300800, offset 0 00000100:00000040:0.0:1775768315.569771:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768315.569775:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768315.569777:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768315.569780:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000200:2.0:1775768315.569782:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000020:00000001:0.0:1775768315.569784:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000400:00000200:2.0:1775768315.569793:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000040:0.0:1775768315.569798:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000400:00000200:2.0:1775768315.569801:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000020:00000001:0.0:1775768315.569805:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768315.569808:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:0.0:1775768315.569810:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000400:00000200:2.0:1775768315.569811:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:0.0:1775768315.569814:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000400:00000200:2.0:1775768315.569819:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000100:00000040:0.0:1775768315.569821:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000200:2.0:1775768315.569826:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1775768315.569829:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1775768315.569833:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00000040:0.0:1775768315.569835:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768315.569838:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000400:00000200:2.0:1775768315.569856:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225523:225523:256:4294967295] 192.168.202.156@tcp LPNI seq info [225519:225519:8:4294967295] 00000400:00000200:2.0:1775768315.569862:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000040:0.0:1775768315.569867:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000100:00080000:0.0:1775768315.569871:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768315.569878:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768315.569880:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 02000000:00000001:0.0:1775768315.569881:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768315.569886:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775768315.569891:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1775768315.569892:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00000001:0.0:1775768315.569896:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775768315.569900:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000010:2.0:1775768315.569900:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000008:00000001:0.0:1775768315.569903:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768315.569906:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768315.569911:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ff100. 00000800:00000200:2.0:1775768315.569911:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000020:00000040:0.0:1775768315.569915:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000800:00000200:2.0:1775768315.569918:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 02000000:00000001:0.0:1775768315.569919:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768315.569921:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768315.569925:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768315.569927:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:2.1:1775768315.569928:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 02000000:00000001:0.0:1775768315.569930:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775768315.569934:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 02000000:00000010:0.0:1775768315.569938:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99422002bc00. 00000800:00000100:2.0:1775768315.569940:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000100:00000001:0.0:1775768315.569944:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1775768315.569945:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000100:00100000:0.0:1775768315.569951:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ff100 x1862022294301056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1775768315.569954:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:0.0:1775768315.569964:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:2.0:1775768315.569965:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000100:00000001:0.0:1775768315.569969:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768315.569972:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000100:00000040:0.0:1775768315.569976:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5e24158] to pc [ptlrpcd_rcv+-1] req@ffff9942099ff100 x1862022294301056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1775768315.569976:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768315.569983:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1775768315.569988:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1775768315.569991:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fe300 x1862022294300800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768340 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768315.569992:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768315.570000:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775768315.570003:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768315.570007:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775768315.570009:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1775768315.570010:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768315.570015:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5567 00000100:00000040:2.0:1775768315.570015:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fe300 x1862022294300800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768340 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775768315.570024:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768315.570031:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775768315.570032:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:2.0:1775768315.570034:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:0.0:1775768315.570036:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775768315.570040:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768315.570041:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768315.570043:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768315.570046:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768315.570049:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:0.0:1775768315.570051:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768315.570052:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:0.0:1775768315.570054:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768315.570057:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000400:00000200:2.0:1775768315.570058:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:0.0:1775768315.570061:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:2.0:1775768315.570063:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4300241725 00000020:00000001:0.0:1775768315.570065:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000040:0.0:1775768315.570069:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000010:2.0:1775768315.570071:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d6c60. 00000100:00000001:0.0:1775768315.570072:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768315.570076:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768315.570078:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:0.0:1775768315.570080:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00000001:2.0:1775768315.570082:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00080000:0.0:1775768315.570083:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:0.0:1775768315.570087:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775768315.570088:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768315.570091:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768315.570091:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4300241725 02000000:00000001:0.0:1775768315.570096:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775768315.570097:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d6000. 00000100:00000001:0.0:1775768315.570099:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775768315.570102:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000008:00100000:0.0:1775768315.570103:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000008:00000001:0.0:1775768315.570107:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768315.570108:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775768315.570109:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775768315.570114:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099fd500. 00000100:00000001:2.0:1775768315.570114:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1775768315.570118:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00000100:00000001:2.0:1775768315.570118:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 02000000:00000001:0.0:1775768315.570121:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768315.570123:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768315.570126:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:2.0:1775768315.570127:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768315/real 1775768315] req@ffff9942099fe300 x1862022294300800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1775768315.570128:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775768315.570130:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768315.570135:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220029800. 00000100:00000001:0.0:1775768315.570140:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1775768315.570145:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00100000:0.0:1775768315.570147:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099fd500 x1862022294301184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:2.0:1775768315.570149:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768315.570153:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000100:00000001:0.0:1775768315.570186:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:2.0:1775768315.570187:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000100:00000001:0.0:1775768315.570190:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775768315.570191:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768315.570196:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:0.0:1775768315.570197:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f246bb2b] to pc [ptlrpcd_rcv+-1] req@ffff9942099fd500 x1862022294301184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768315.570204:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fe300 x1862022294300800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768315.570212:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775768315.570216:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768315.570221:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775768315.570222:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775768315.570225:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:2.0:1775768315.570225:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fe300 x1862022294300800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768315.570229:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775768315.570233:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5567 00000400:00000200:0.0:1775768315.570240:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768315.570241:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:0.0:1775768315.570244:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:2.0:1775768315.570244:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1775768315.570246:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1775768315.570248:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:0.0:1775768315.570249:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768315.570252:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775768315.570252:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775768315.570256:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768315.570257:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00100000:2.0:1775768315.570260:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fe300 x1862022294300800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775768315.570261:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775768315.570263:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 00000020:00000001:0.0:1775768315.570266:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775768315.570269:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 00000020:00000001:0.0:1775768315.570275:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000001:2.0:1775768315.570275:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1775768315.570279:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000100:00000040:2.0:1775768315.570284:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fe300 x1862022294300800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768340 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775768315.570287:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1775768315.570293:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775768315.570298:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768315.570298:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768315.570303:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:2.0:1775768315.570303:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768315.570307:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000040:0.0:1775768315.570309:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00000001:2.0:1775768315.570309:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768315.570314:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1775768315.570315:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1775768315.570318:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775768315.570319:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768315.570324:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:0.0:1775768315.570325:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775768315.570330:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1775768315.570333:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 02000000:00000001:0.0:1775768315.570334:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768315.570337:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775768315.570338:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000008:00100000:0.0:1775768315.570339:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000008:00000001:0.0:1775768315.570343:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768315.570345:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768315.570347:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fe300 x1862022294300800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768340 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1775768315.570349:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9942099ff800. 00000020:00000040:0.0:1775768315.570352:0:13471:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:0.0:1775768315.570357:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775768315.570359:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768315.570362:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775768315.570364:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:2.0:1775768315.570365:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294300800:192.168.202.156@tcp:250:kworker.0 02000000:00000001:0.0:1775768315.570366:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775768315.570372:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff994220029400. 00000100:00000001:2.0:1775768315.570373:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768315.570375:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768315.570377:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00100000:0.0:1775768315.570381:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9942099ff800 x1862022294301312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:2.0:1775768315.570383:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fe300 x1862022294300800/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768340 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775768315.570398:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775768315.570402:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768315.570403:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 00000100:00000040:0.0:1775768315.570407:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000aa4e8b71] to pc [ptlrpcd_rcv+-1] req@ffff9942099ff800 x1862022294301312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1775768315.570408:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768315.570411:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1e800. 00000100:00000001:0.0:1775768315.570415:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768315.570418:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000040:0.0:1775768315.570420:0:13471:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5578) 00000020:00000001:2.0:1775768315.570421:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768315.570424:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:2.0:1775768315.570427:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768315.570430:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220028400. 02000000:00000001:2.0:1775768315.570441:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768315.570443:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768315.570445:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fe300. 00000100:00000001:2.0:1775768315.570449:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768315.570450:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768315.570453:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768315.570460:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768315.570462:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768315.570464:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768315.570466:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768315.570467:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768315.570471:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fc700 x1862022294300928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768315.570478:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768315.570479:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768315.570481:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768315.570483:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768315.570485:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768315.570488:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294300928:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768315.570491:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768315.570493:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768315.570494:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768315.570497:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768315.570500:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1e800. 02000000:00000001:2.0:1775768315.570501:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768315.570505:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768315.570508:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768315.570511:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768315.570514:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294300928, portal 10 00000100:00000001:2.0:1775768315.570516:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768315.570517:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641198336 : -112966068353280 : ffff9942099fc700) 00000100:00000040:2.0:1775768315.570523:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fc700 x1862022294300928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768340 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768315.570529:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768315.570531:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768315.570533:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768315.570535:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294300928, offset 0 00000400:00000200:2.0:1775768315.570538:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768315.570542:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768315.570545:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768315.570548:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768315.570550:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768315.570552:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768315.570555:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768315.570558:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225524:225524:256:4294967295] 192.168.202.156@tcp LPNI seq info [225520:225520:8:4294967295] 00000400:00000200:2.0:1775768315.570561:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768315.570568:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768315.570573:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768315.570576:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768315.570579:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768315.570582:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1775768315.570586:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768315.570589:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000100:2.0:1775768315.570592:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768315.570594:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768315.570596:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768315.570599:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768315.570602:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768315.570603:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768315.570605:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768315.570608:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fc700 x1862022294300928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768340 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768315.570615:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768315.570617:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fc700 x1862022294300928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768340 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768315.570623:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768315.570624:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768315.570627:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768315.570630:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:2.0:1775768315.570633:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768315.570635:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300240403 00000100:00000001:2.0:1775768315.570638:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300241725 00000100:00000001:2.0:1775768315.570640:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300240403 00000100:00000001:2.0:1775768315.570643:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300241725 00000100:00000001:2.0:1775768315.570645:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768315.570647:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768315.570649:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768315.570652:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768315/real 1775768315] req@ffff9942099fc700 x1862022294300928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768315.570659:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768315.570660:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768315.570661:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768315.570663:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768315.570665:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768315.570666:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768315.570669:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fc700 x1862022294300928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768315.570676:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fc700 x1862022294300928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768315.570682:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768315.570683:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768315.570684:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768315.570686:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768315.570689:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fc700 x1862022294300928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768315.570694:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768315.570697:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fc700 x1862022294300928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768340 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768315.570702:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768315.570705:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768315.570706:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768315.570707:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768315.570708:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768315.570711:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768315.570713:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768315.570716:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768315.570718:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768315.570721:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fc700 x1862022294300928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768340 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768315.570728:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294300928:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768315.570732:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768315.570735:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ff100 x1862022294301056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768315.570746:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768315.570747:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768315.570749:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768315.570750:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768315.570752:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768315.570755:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294301056:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768315.570758:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768315.570760:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768315.570762:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768315.570763:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768315.570766:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f000. 02000000:00000001:2.0:1775768315.570768:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768315.570771:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768315.570773:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768315.570775:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768315.570778:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294301056, portal 10 00000100:00000001:2.0:1775768315.570780:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768315.570781:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641209088 : -112966068342528 : ffff9942099ff100) 00000100:00000040:2.0:1775768315.570787:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ff100 x1862022294301056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768340 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768315.570793:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768315.570795:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768315.570798:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768315.570800:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294301056, offset 0 00000400:00000200:2.0:1775768315.570804:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768315.570807:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768315.570811:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768315.570813:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768315.570816:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768315.570819:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768315.570821:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768315.570824:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225525:225525:256:4294967295] 192.168.202.156@tcp LPNI seq info [225521:225521:8:4294967295] 00000400:00000200:2.0:1775768315.570828:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768315.570834:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768315.570838:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768315.570853:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768315.570857:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768315.570859:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1775768315.570861:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768315.570863:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000100:2.0:1775768315.570865:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768315.570867:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768315.570869:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768315.570871:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768315.570873:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768315.570874:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768315.570876:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768315.570878:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ff100 x1862022294301056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768340 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768315.570884:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768315.570885:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ff100 x1862022294301056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768340 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768315.570889:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768315.570890:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768315.570892:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768315.570894:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:2.0:1775768315.570896:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768315.570898:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300240395 00000100:00000001:2.0:1775768315.570900:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300241725 00000100:00000001:2.0:1775768315.570901:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300240395 00000100:00000001:2.0:1775768315.570902:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300241725 00000100:00000001:2.0:1775768315.570903:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768315.570905:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768315.570906:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768315.570908:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768315/real 1775768315] req@ffff9942099ff100 x1862022294301056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768315.570913:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768315.570914:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768315.570915:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768315.570916:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768315.570917:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768315.570918:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768315.570920:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ff100 x1862022294301056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768315.570925:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ff100 x1862022294301056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768315.570928:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768315.570929:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768315.570930:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768315.570931:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768315.570932:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ff100 x1862022294301056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768315.570936:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768315.570938:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ff100 x1862022294301056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768340 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768315.570941:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768315.570942:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768315.570943:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768315.570944:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768315.570945:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768315.570946:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768315.570948:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768315.570950:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768315.570951:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768315.570954:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ff100 x1862022294301056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768340 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768315.570958:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294301056:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768315.570960:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768315.570961:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099fd500 x1862022294301184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768315.570965:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768315.570965:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768315.570967:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768315.570968:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768315.570969:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768315.570970:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294301184:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768315.570972:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768315.570973:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768315.570973:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768315.570975:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768315.570977:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1d000. 02000000:00000001:2.0:1775768315.570978:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768315.570980:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768315.570982:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768315.570984:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768315.570986:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294301184, portal 4 00000100:00000001:2.0:1775768315.570987:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768315.570988:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641201920 : -112966068349696 : ffff9942099fd500) 00000100:00000040:2.0:1775768315.570991:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099fd500 x1862022294301184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768340 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768315.570995:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768315.570996:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768315.570997:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768315.570999:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294301184, offset 0 00000400:00000200:2.0:1775768315.571001:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768315.571003:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768315.571006:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768315.571008:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768315.571009:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768315.571011:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768315.571012:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768315.571014:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225526:225526:256:4294967295] 192.168.202.156@tcp LPNI seq info [225522:225522:8:4294967295] 00000400:00000200:2.0:1775768315.571017:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768315.571020:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768315.571023:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768315.571024:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768315.571026:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768315.571028:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1775768315.571030:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768315.571032:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000100:2.0:1775768315.571034:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768315.571035:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768315.571037:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768315.571039:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768315.571041:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768315.571042:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768315.571043:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768315.571045:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099fd500 x1862022294301184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768340 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768315.571049:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768315.571051:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099fd500 x1862022294301184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768340 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768315.571055:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768315.571055:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768315.571057:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768315.571059:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:2.0:1775768315.571061:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768315.571062:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300240418 00000100:00000001:2.0:1775768315.571064:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300241725 00000100:00000001:2.0:1775768315.571065:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300240418 00000100:00000001:2.0:1775768315.571066:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300241725 00000100:00000001:2.0:1775768315.571067:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768315.571068:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768315.571069:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768315.571071:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768315/real 1775768315] req@ffff9942099fd500 x1862022294301184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768315.571075:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768315.571075:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768315.571076:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768315.571077:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768315.571078:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768315.571080:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768315.571081:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099fd500 x1862022294301184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768315.571086:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099fd500 x1862022294301184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768315.571089:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768315.571090:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768315.571091:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768315.571091:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768315.571093:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099fd500 x1862022294301184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768315.571096:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768315.571098:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099fd500 x1862022294301184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768340 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768315.571103:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768315.571104:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768315.571105:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768315.571106:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768315.571107:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768315.571110:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768315.571111:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768315.571113:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768315.571114:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768315.571115:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099fd500 x1862022294301184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768340 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768315.571119:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294301184:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768315.571121:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768315.571123:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9942099ff800 x1862022294301312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768315.571127:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768315.571128:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768315.571129:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768315.571130:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768315.571131:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768315.571132:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9942099ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294301312:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768315.571134:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768315.571135:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768315.571135:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768315.571136:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768315.571138:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1ec00. 02000000:00000001:2.0:1775768315.571140:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768315.571143:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768315.571146:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768315.571148:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768315.571151:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294301312, portal 4 00000100:00000001:2.0:1775768315.571153:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768315.571154:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107641210880 : -112966068340736 : ffff9942099ff800) 00000100:00000040:2.0:1775768315.571178:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9942099ff800 x1862022294301312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768340 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768315.571184:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768315.571186:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768315.571189:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768315.571191:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294301312, offset 0 00000400:00000200:2.0:1775768315.571195:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768315.571198:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768315.571202:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768315.571204:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768315.571207:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768315.571209:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768315.571211:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768315.571213:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225527:225527:256:4294967295] 192.168.202.156@tcp LPNI seq info [225523:225523:8:4294967295] 00000400:00000200:2.0:1775768315.571216:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768315.571220:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768315.571222:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768315.571225:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768315.571227:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768315.571229:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1775768315.571232:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768315.571233:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 47 secs later) 00000800:00000100:2.0:1775768315.571235:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768315.571237:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768315.571239:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768315.571241:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768315.571243:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768315.571244:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768315.571246:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768315.571248:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9942099ff800 x1862022294301312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768340 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768315.571253:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768315.571254:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9942099ff800 x1862022294301312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768340 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768315.571258:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768315.571259:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768315.571260:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768315.571262:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:2.0:1775768315.571264:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768315.571266:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300240422 00000100:00000001:2.0:1775768315.571267:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300241725 00000100:00000001:2.0:1775768315.571269:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300240422 00000100:00000001:2.0:1775768315.571270:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300241725 00000100:00000001:2.0:1775768315.571271:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768315.571272:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768315.571273:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768315.571275:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768315/real 1775768315] req@ffff9942099ff800 x1862022294301312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768315.571279:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768315.571280:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768315.571281:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768315.571282:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768315.571283:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768315.571284:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768315.571286:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9942099ff800 x1862022294301312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768315.571291:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9942099ff800 x1862022294301312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768315.571298:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768315.571299:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768315.571300:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768315.571301:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768315.571303:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9942099ff800 x1862022294301312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768315.571306:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768315.571308:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9942099ff800 x1862022294301312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768340 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768315.571311:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768315.571313:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768315.571313:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768315.571314:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768315.571315:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768315.571318:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768315.571319:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768315.571321:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768315.571322:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768315.571323:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9942099ff800 x1862022294301312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768340 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768315.571327:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9942099ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294301312:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768315.571329:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768315.571330:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768315.571332:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fc700 x1862022294300928/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768340 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768315.571336:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768315.571337:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768315.571339:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1e800. 02000000:00000001:2.0:1775768315.571340:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768315.571341:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768315.571342:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:2.0:1775768315.571344:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768315.571345:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002b800. 02000000:00000001:2.0:1775768315.571347:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768315.571348:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768315.571348:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fc700. 00000100:00000001:2.0:1775768315.571350:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768315.571351:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768315.571352:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768315.571354:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ff100 x1862022294301056/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768340 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768315.571358:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768315.571358:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768315.571359:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f000. 02000000:00000001:2.0:1775768315.571360:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768315.571361:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768315.571362:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:2.0:1775768315.571363:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768315.571363:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99422002bc00. 02000000:00000001:2.0:1775768315.571364:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768315.571365:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768315.571366:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ff100. 00000100:00000001:2.0:1775768315.571367:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768315.571367:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768315.571368:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768315.571370:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099fd500 x1862022294301184/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768340 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768315.571373:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768315.571373:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768315.571374:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1d000. 02000000:00000001:2.0:1775768315.571376:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768315.571377:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768315.571377:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:2.0:1775768315.571379:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768315.571380:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029800. 02000000:00000001:2.0:1775768315.571381:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768315.571382:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768315.571382:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099fd500. 00000100:00000001:2.0:1775768315.571384:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768315.571384:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768315.571385:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768315.571386:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9942099ff800 x1862022294301312/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768340 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768315.571389:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768315.571390:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768315.571391:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1ec00. 02000000:00000001:2.0:1775768315.571392:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768315.571392:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768315.571393:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:2.0:1775768315.571394:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768315.571394:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff994220029400. 02000000:00000001:2.0:1775768315.571395:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768315.571396:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768315.571396:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9942099ff800. 00000100:00000001:2.0:1775768315.571397:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768315.571398:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768315.571399:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768315.571401:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768315.571402:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768315.571403:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768315.571404:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768315.633262:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768315.633268:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768315.633273:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768315.633274:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768315.633277:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768315.633278:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768315.633286:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768315.633287:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768315.633291:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768315.633292:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768315.633400:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768315.633403:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768315.633406:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768315.697318:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768315.697329:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768315.697332:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768315.697340:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768315.697344:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768315.697349:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768315.697352:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768315.697356:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768315.697360:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768316.593271:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768316.593271:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768316.593271:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768316.593278:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768316.593278:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768316.593279:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768316.593281:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768316.593282:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768316.593287:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775768316.593288:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:2.0:1775768316.593289:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768316.593292:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768316.593293:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:2.0:1775768316.593295:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:1.0:1775768316.593296:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768316.593297:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768316.593300:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768316.593302:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768316.593303:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768316.593306:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768316.593311:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768316.593312:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768316.593315:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768316.593318:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768316.593319:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768316.593321:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768316.593325:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768316.657319:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768316.657328:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768316.657335:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768316.657337:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768316.657341:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768316.657342:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768316.657346:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768316.657347:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768316.657351:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768316.657353:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768316.657402:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768316.657405:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768316.657407:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768316.721296:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768316.721304:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768316.721306:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768316.721312:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768316.721316:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768316.721321:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768316.721324:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768316.721326:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768316.721329:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775768316.913295:0:15036:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc800 (cpu 0) has reached its deadline 4300241725, at 4300241728 00000100:00000040:1.0:1775768316.913295:0:15013:0:(connection.c:75:cpu_latency_work()) work item of ffff9942108bc910 (cpu 0) has reached its deadline 4300241725, at 4300241728 00000100:00000010:0.0:1775768316.913313:0:15036:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d6c60. 00000100:00000010:1.0:1775768316.913315:0:15013:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9942182d6000. 00000100:00000001:1.0:1775768317.617234:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768317.617234:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768317.617238:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768317.617242:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768317.617242:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768317.617244:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768317.617244:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768317.617245:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768317.617247:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768317.617252:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:2.0:1775768317.617253:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775768317.617253:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:1.0:1775768317.617258:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768317.617259:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768317.617259:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768317.617263:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768317.617263:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768317.617264:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768317.617268:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768317.617271:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768317.617275:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768317.617277:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768317.617280:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768317.617284:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768317.617285:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768317.617288:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768317.617291:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768317.681295:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768317.681304:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768317.681312:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768317.681316:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768317.681319:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768317.681323:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768317.681326:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768317.681328:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768317.681330:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768317.681332:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768317.681335:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768317.681336:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768317.681337:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768317.681338:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768317.681339:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768317.681340:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768317.681342:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768317.681343:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768317.681345:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768317.681346:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768317.681347:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768317.681347:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768317.681348:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768317.681349:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768317.681350:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768317.681351:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768317.681353:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768317.681354:0:15036:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775768317.681355:0:15036:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775768317.681355:0:15036:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768317.681356:0:15036:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775768317.681357:0:15036:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768317.681358:0:15036:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768317.681359:0:15036:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768317.681421:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768317.681426:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768317.681430:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768317.745380:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768317.745389:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768317.745391:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768317.745399:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768317.745403:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768317.745408:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768317.745411:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768317.745414:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768317.745417:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768318.641280:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768318.641280:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768318.641288:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768318.641288:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768318.641291:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768318.641291:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768318.641298:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775768318.641298:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768318.641303:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768318.641304:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768318.641306:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768318.641308:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768318.641308:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768318.641312:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768318.641312:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768318.641315:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768318.641316:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775768318.641321:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:2.0:1775768318.641321:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768318.641321:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768318.641325:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768318.641326:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768318.641330:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768318.641330:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768318.641335:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768318.641337:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768318.641339:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768318.705303:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768318.705310:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768318.705315:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768318.705316:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768318.705320:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768318.705322:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768318.705325:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768318.705328:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768318.705331:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768318.705332:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768318.705348:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768318.705351:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768318.705353:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768318.769249:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768318.769261:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768318.769262:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768318.769269:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768318.769273:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768318.769278:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768318.769282:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768318.769284:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768318.769295:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768319.665280:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768319.665280:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768319.665287:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768319.665287:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768319.665289:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768319.665289:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768319.665296:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00100000:3.0:1775768319.665296:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00000001:3.0:1775768319.665300:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768319.665301:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768319.665305:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768319.665306:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768319.665309:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775768319.665312:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768319.665314:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768319.665317:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768319.665320:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768319.665323:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768319.665405:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768319.665407:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768319.665408:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775768319.665410:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00000001:0.0:1775768319.665412:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775768319.665413:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768319.665415:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768319.665416:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775768319.665417:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768319.729291:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768319.729299:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768319.729306:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768319.729307:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768319.729311:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768319.729312:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768319.729316:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768319.729317:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768319.729320:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768319.729321:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775768319.729374:0:14473:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775768319.729378:0:14473:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775768319.729381:0:14473:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768319.793284:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768319.793297:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768319.793299:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768319.793305:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768319.793308:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768319.793312:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768319.793315:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768319.793317:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768319.793319:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775768320.689308:0:14454:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775768320.689308:0:6169:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775768320.689316:0:14454:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775768320.689316:0:6169:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775768320.689319:0:14454:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768320.689319:0:6169:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768320.689322:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 97b822a0-1031-438a-b417-b0f1b9d1e3df->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775768320.689326:0:14454:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4e40 going to sleep for 0 seconds 00000100:00100000:3.0:1775768320.689326:0:6169:0:(client.c:2691:ptlrpc_set_wait()) set ffff99421812f000 going to sleep for 0 seconds 00000100:00080000:1.0:1775768320.689329:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775768320.689332:0:14454:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775768320.689332:0:6169:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768320.689333:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1775768320.689336:0:14454:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768320.689337:0:6169:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1775768320.689338:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775768320.689340:0:14454:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775768320.689345:0:14454:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768320.689346:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:0.0:1775768320.689349:0:14454:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768320.689356:0:13471:0:(import.c:509:import_select_connection()) MGC192.168.202.156@tcp: connect to NID 192.168.202.156@tcp last attempt 5573 00000400:00000200:1.0:1775768320.689369:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768320.689376:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775768320.689380:0:13471:0:(import.c:588:import_select_connection()) MGC192.168.202.156@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775768320.689384:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768320.689387:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768320.689390:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768320.689392:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775768320.689396:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775768320.689397:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e678d 00000020:00000001:1.0:1775768320.689399:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775768320.689402:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994210d1d000 refcount=3 00000020:00000001:1.0:1775768320.689403:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000020:00000001:1.0:1775768320.689405:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107761917952 : -112965947633664 : ffff994210d1d000) 00000100:00000040:1.0:1775768320.689408:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768320.689410:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768320.689412:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768320.689413:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775768320.689416:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994210d1d000 : new refcount 2 00000100:00080000:1.0:1775768320.689421:0:13471:0:(import.c:623:import_select_connection()) MGC192.168.202.156@tcp: import ffff994205567800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775768320.689423:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768320.689426:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775768320.689430:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768320.689434:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768320.689437:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775768320.689439:0:13471:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768320.689442:0:14453:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775768320.689446:0:14453:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768320.689448:0:14453:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768320.689451:0:14453:0:(client.c:2691:ptlrpc_set_wait()) set ffff99420900b180 going to sleep for 0 seconds 00000100:00000001:2.0:1775768320.689453:0:14453:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768320.689455:0:14453:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:1.0:1775768320.689457:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207888700. 00000020:00000040:1.0:1775768320.689462:0:13471:0:(genops.c:1229:class_import_get()) import ffff994205567800 refcount=4 obd=MGC192.168.202.156@tcp 02000000:00000001:1.0:1775768320.689467:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768320.689468:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768320.689472:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775768320.689475:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1775768320.689479:0:6169:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 02000000:00000001:1.0:1775768320.689480:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775768320.689482:0:6169:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775768320.689486:0:6169:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775768320.689492:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543d400. 00000100:00000001:1.0:1775768320.689500:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768320.689509:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207888700 x1862022294301440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775768320.689519:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775768320.689520:0:14453:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768320.689523:0:14453:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768320.689524:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768320.689526:0:14453:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768320.689529:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009f2ba301] to pc [ptlrpcd_rcv+-1] req@ffff994207888700 x1862022294301440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768320.689541:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768320.689544:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768320.689547:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775768320.689548:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775768320.689550:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768320.689552:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775768320.689552:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:1.0:1775768320.689556:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5573 00000100:00000001:2.0:1775768320.689556:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000400:00000200:1.0:1775768320.689560:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768320.689561:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000400:00000200:1.0:1775768320.689564:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775768320.689566:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775768320.689569:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207888700 x1862022294301440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775768320.689571:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768320.689575:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768320.689578:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768320.689581:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775768320.689585:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775768320.689587:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e1 00000100:00000001:2.0:1775768320.689588:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000001:1.0:1775768320.689591:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:2.0:1775768320.689591:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1775768320.689593:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994205748800 refcount=5 02000000:00000001:2.0:1775768320.689595:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:1.0:1775768320.689596:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 00000020:00000001:1.0:1775768320.689599:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571255296 : -112966138296320 : ffff994205748800) 02000000:00000001:2.0:1775768320.689599:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768320.689605:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768320.689606:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768320.689609:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:2.0:1775768320.689610:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207888700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294301440:192.168.202.156@tcp:250:kworker.0 00000100:00000040:1.0:1775768320.689613:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768320.689618:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:2.0:1775768320.689618:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000040:1.0:1775768320.689621:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994205748800 : new refcount 4 02000000:00000001:2.0:1775768320.689622:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:1.0:1775768320.689626:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9942077a7000: import ffff99421033b800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 02000000:00000001:2.0:1775768320.689626:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768320.689630:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768320.689631:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1775768320.689632:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000010:2.0:1775768320.689637:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1ec00. 02000000:00000001:1.0:1775768320.689638:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768320.689639:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768320.689640:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768320.689643:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775768320.689647:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000400:00000200:2.0:1775768320.689647:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000008:00000001:1.0:1775768320.689651:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775768320.689652:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000100:00000001:1.0:1775768320.689653:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775768320.689659:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207889880. 00000020:00000040:1.0:1775768320.689660:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033b800 refcount=24 obd=lustre-MDT0000-mdc-ffff9942077a7000 02000000:00000001:1.0:1775768320.689662:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000010:2.0:1775768320.689662:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 02000000:00000001:1.0:1775768320.689664:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768320.689666:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775768320.689667:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775768320.689668:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775768320.689671:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543e800. 00000100:00000200:2.0:1775768320.689672:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294301440, portal 25 00000100:00000001:1.0:1775768320.689674:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768320.689677:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00100000:1.0:1775768320.689679:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207889880 x1862022294301568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775768320.689680:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606120192 : -112966103431424 : ffff994207888700) 00000100:00000040:2.0:1775768320.689690:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207888700 x1862022294301440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768345 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768320.689708:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775768320.689709:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1775768320.689713:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768320.689714:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768320.689718:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000040:1.0:1775768320.689722:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f11856b8] to pc [ptlrpcd_rcv+-1] req@ffff994207889880 x1862022294301568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768320.689723:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022294301440, offset 0 00000400:00000200:2.0:1775768320.689733:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00000001:1.0:1775768320.689740:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768320.689742:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:2.0:1775768320.689746:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00080000:1.0:1775768320.689748:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775768320.689751:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775768320.689752:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:2.0:1775768320.689754:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00000001:1.0:1775768320.689756:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775768320.689761:0:13471:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5573 00000400:00000200:2.0:1775768320.689761:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775768320.689768:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768320.689768:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1775768320.689772:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775768320.689775:0:13471:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1775768320.689775:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768320.689781:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00000040:1.0:1775768320.689782:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768320.689784:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768320.689788:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000200:2.0:1775768320.689789:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225528:225528:256:4294967295] 192.168.202.156@tcp LPNI seq info [225524:225524:8:4294967295] 00000100:00000001:1.0:1775768320.689803:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775768320.689806:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775768320.689806:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67e8 00000020:00000001:1.0:1775768320.689807:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775768320.689809:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff99420574e000 refcount=5 00000020:00000001:1.0:1775768320.689814:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000400:00000200:2.0:1775768320.689818:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000020:00000001:1.0:1775768320.689819:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107571277824 : -112966138273792 : ffff99420574e000) 00000100:00000040:1.0:1775768320.689824:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768320.689827:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768320.689833:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000400:00000200:2.0:1775768320.689835:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000100:00000001:1.0:1775768320.689839:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775768320.689845:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff99420574e000 : new refcount 4 00000800:00000200:2.0:1775768320.689845:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00080000:1.0:1775768320.689851:0:13471:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9942077a7000: import ffff99421033e000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000800:00000010:2.0:1775768320.689854:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000100:00000001:1.0:1775768320.689855:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768320.689859:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775768320.689862:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768320.689867:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1775768320.689867:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000100:00000001:1.0:1775768320.689872:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000800:00000200:2.0:1775768320.689874:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000008:00100000:1.0:1775768320.689875:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775768320.689881:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768320.689886:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775768320.689886:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768320.689891:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000100:00000010:1.0:1775768320.689892:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff99420788b100. 00000020:00000040:1.0:1775768320.689897:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e000 refcount=67 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000800:00000100:2.0:1775768320.689900:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 02000000:00000001:1.0:1775768320.689902:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000800:00000010:2.0:1775768320.689904:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 02000000:00000001:1.0:1775768320.689905:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768320.689908:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775768320.689910:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775768320.689912:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768320.689914:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000010:1.0:1775768320.689916:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543f000. 00000400:00000200:2.0:1775768320.689921:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000100:00000001:1.0:1775768320.689922:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768320.689928:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000100:00100000:1.0:1775768320.689929:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff99420788b100 x1862022294301696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:2.0:1775768320.689933:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:1.0:1775768320.689939:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775768320.689941:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768320.689942:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:1.0:1775768320.689945:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e26cf977] to pc [ptlrpcd_rcv+-1] req@ffff99420788b100 x1862022294301696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768320.689950:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207888700 x1862022294301440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768345 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768320.689953:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775768320.689957:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768320.689963:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775768320.689967:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775768320.689970:0:13471:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775768320.689972:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1775768320.689973:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00000040:2.0:1775768320.689977:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207888700 x1862022294301440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 0 dl 1775768345 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768320.689980:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5573 00000100:00000001:2.0:1775768320.689988:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775768320.689989:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768320.689991:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:1.0:1775768320.689994:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00080000:1.0:1775768320.689998:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1775768320.689998:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775768320.690004:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000400:00000200:2.0:1775768320.690006:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:1.0:1775768320.690010:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775768320.690010:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4300246845 00000100:00000040:1.0:1775768320.690015:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000010:2.0:1775768320.690016:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d6240. 00000100:00000001:1.0:1775768320.690018:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000001:1.0:1775768320.690022:0:13471:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775768320.690023:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67f6 00000020:00000001:1.0:1775768320.690024:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:2.0:1775768320.690024:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000020:00000040:1.0:1775768320.690028:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206082000 refcount=4 00000100:00000001:2.0:1775768320.690031:0:2399:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4300246845 00000020:00000001:1.0:1775768320.690032:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000010:2.0:1775768320.690034:0:2399:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9942182d64e0. 00000100:00000001:2.0:1775768320.690038:0:2399:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000020:00000001:1.0:1775768320.690047:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580928000 : -112966128623616 : ffff994206082000) 00000100:00000001:2.0:1775768320.690049:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1775768320.690054:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1775768320.690055:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768320.690059:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775768320.690060:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000040:1.0:1775768320.690063:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768320.690065:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000020:00000040:1.0:1775768320.690068:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206082000 : new refcount 3 00000100:00100000:2.0:1775768320.690069:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768320/real 1775768320] req@ffff994207888700 x1862022294301440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768320.690072:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9942077a7000: import ffff99421033e800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775768320.690076:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775768320.690080:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775768320.690084:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775768320.690087:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1775768320.690088:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1775768320.690089:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000001:2.0:1775768320.690090:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000008:00100000:1.0:1775768320.690093:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1253376, lost: 0. 00000400:00000010:2.0:1775768320.690094:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000008:00000001:1.0:1775768320.690098:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768320.690100:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000100:00000001:1.0:1775768320.690102:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775768320.690104:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000010:1.0:1775768320.690108:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207889f80. 00000100:00000001:2.0:1775768320.690108:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000040:1.0:1775768320.690112:0:13471:0:(genops.c:1229:class_import_get()) import ffff99421033e800 refcount=17 obd=lustre-OST0000-osc-ffff9942077a7000 00000100:00000200:2.0:1775768320.690121:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207888700 x1862022294301440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1775768320.690125:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775768320.690128:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768320.690131:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775768320.690133:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775768320.690135:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1775768320.690137:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207888700 x1862022294301440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1775768320.690140:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543f800. 00000100:00000001:1.0:1775768320.690147:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768320.690149:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768320.690152:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768320.690180:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768320.690182:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768320.690188:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207888700 x1862022294301440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768320.690195:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768320.690200:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207888700 x1862022294301440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768345 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768320.690202:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207889f80 x1862022294301824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775768320.690221:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775768320.690224:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775768320.690227:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768320.690228:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768320.690232:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768320.690234:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775768320.690236:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ed2274f4] to pc [ptlrpcd_rcv+-1] req@ffff994207889f80 x1862022294301824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768320.690238:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768320.690241:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768320.690248:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994205567800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:1.0:1775768320.690253:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775768320.690257:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.156@tcp failed (-110) 00000100:00080000:1.0:1775768320.690258:0:13471:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c72e6f0-c073-4915-a1e7-5dde21a8c485->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775768320.690261:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00080000:1.0:1775768320.690263:0:13471:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775768320.690266:0:13471:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000040:2.0:1775768320.690269:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207888700 x1862022294301440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768345 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768320.690270:0:13471:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775768320.690276:0:13471:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775768320.690281:0:13471:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: connect to NID 192.168.202.156@tcp last attempt 5573 00000100:00100000:2.0:1775768320.690287:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207888700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:97b822a0-1031-438a-b417-b0f1b9d1e3df:2399:1862022294301440:192.168.202.156@tcp:250:kworker.0 00000400:00000200:1.0:1775768320.690289:0:13471:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:1.0:1775768320.690298:0:13471:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000100:00000001:2.0:1775768320.690298:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768320.690303:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00080000:1.0:1775768320.690304:0:13471:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775768320.690310:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000100:00000040:2.0:1775768320.690310:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207888700 x1862022294301440/t0(0) o250->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768345 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775768320.690317:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775768320.690322:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768320.690327:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 00000100:00000001:2.0:1775768320.690327:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 00000020:00000001:1.0:1775768320.690331:0:13471:0:(genops.c:811:class_conn2export()) Process entered 02000000:00000001:2.0:1775768320.690331:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000040:1.0:1775768320.690333:0:13471:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e144250943e67fd 02000000:00000010:2.0:1775768320.690335:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1ec00. 00000020:00000001:1.0:1775768320.690336:0:13471:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775768320.690340:0:13471:0:(lustre_handles.c:131:class_handle2object()) GET export ffff994206080000 refcount=4 02000000:00000001:2.0:1775768320.690342:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775768320.690343:0:13471:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000001:2.0:1775768320.690346:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000001:1.0:1775768320.690347:0:13471:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446631107580919808 : -112966128631808 : ffff994206080000) 00000020:00000040:2.0:1775768320.690349:0:2399:0:(genops.c:1242:class_import_put()) import ffff994205567800 refcount=3 obd=MGC192.168.202.156@tcp 00000100:00000040:1.0:1775768320.690353:0:13471:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff99422086aa40 refcount 14 to 192.168.202.156@tcp 00000020:00000001:2.0:1775768320.690354:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1775768320.690357:0:13471:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000010:2.0:1775768320.690359:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543d400. 00000100:00000040:1.0:1775768320.690362:0:13471:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff99422086aa40 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1775768320.690365:0:13471:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446631108025428544 : -112965684123072 : ffff99422086aa40) 02000000:00000001:2.0:1775768320.690365:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768320.690369:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000020:00000040:1.0:1775768320.690370:0:13471:0:(genops.c:898:class_export_put()) PUTting export ffff994206080000 : new refcount 3 00000100:00000010:2.0:1775768320.690372:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207888700. 00000100:00080000:1.0:1775768320.690374:0:13471:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9942077a7000: import ffff994206081800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1775768320.690377:0:13471:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768320.690377:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 02000000:00000001:1.0:1775768320.690381:0:13471:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1775768320.690381:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1775768320.690384:0:13471:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768320.690385:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1775768320.690388:0:13471:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768320.690391:0:13471:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775768320.690393:0:13471:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 1282048, lost: 0. 00000100:00000001:2.0:1775768320.690395:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000008:00000001:1.0:1775768320.690396:0:13471:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768320.690398:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768320.690400:0:13471:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768320.690402:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000010:1.0:1775768320.690405:0:13471:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff994207888e00. 00000100:00000001:2.0:1775768320.690405:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768320.690408:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000040:1.0:1775768320.690410:0:13471:0:(genops.c:1229:class_import_get()) import ffff994206081800 refcount=18 obd=lustre-OST0001-osc-ffff9942077a7000 02000000:00000001:1.0:1775768320.690414:0:13471:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000040:2.0:1775768320.690415:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207889880 x1862022294301568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1775768320.690417:0:13471:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768320.690421:0:13471:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775768320.690424:0:13471:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775768320.690427:0:13471:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768320.690430:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768320.690433:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775768320.690434:0:13471:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff99420543fc00. 02000000:00000001:2.0:1775768320.690436:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1775768320.690439:0:13471:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768320.690439:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768320.690444:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768320.690450:0:13471:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff994207888e00 x1862022294301952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768320.690451:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207889880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294301568:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768320.690459:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768320.690463:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768320.690466:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775768320.690467:0:13471:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1775768320.690469:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1775768320.690470:0:13471:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775768320.690474:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1ec00. 00000100:00000040:1.0:1775768320.690478:0:13471:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001812f966] to pc [ptlrpcd_rcv+-1] req@ffff994207888e00 x1862022294301952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1775768320.690478:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768320.690486:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768320.690491:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000100:00000001:1.0:1775768320.690498:0:13471:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775768320.690498:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768320.690504:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294301568, portal 10 00000100:00000040:1.0:1775768320.690505:0:13471:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (5583) 00000100:00000001:2.0:1775768320.690507:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768320.690511:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606124672 : -112966103426944 : ffff994207889880) 00000100:00000040:2.0:1775768320.690519:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207889880 x1862022294301568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768345 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768320.690526:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768320.690529:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768320.690532:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768320.690535:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294301568, offset 0 00000400:00000200:2.0:1775768320.690540:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768320.690544:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768320.690548:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768320.690550:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768320.690553:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768320.690555:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768320.690557:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768320.690560:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225529:225529:256:4294967295] 192.168.202.156@tcp LPNI seq info [225525:225525:8:4294967295] 00000400:00000200:2.0:1775768320.690564:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768320.690570:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768320.690575:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768320.690578:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768320.690582:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768320.690585:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775768320.690590:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768320.690592:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000100:2.0:1775768320.690596:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768320.690598:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768320.690601:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768320.690604:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768320.690607:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768320.690609:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768320.690612:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768320.690615:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207889880 x1862022294301568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768345 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768320.690623:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768320.690626:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207889880 x1862022294301568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768345 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768320.690633:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768320.690635:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768320.690638:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768320.690642:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: using a timeout of 17893 usecs (18 jiffies) 00000400:00000200:2.0:1775768320.690646:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768320.690647:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300245523 00000100:00000001:2.0:1775768320.690650:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300246845 00000100:00000001:2.0:1775768320.690653:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300245523 00000100:00000001:2.0:1775768320.690655:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300246845 00000100:00000001:2.0:1775768320.690657:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768320.690659:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768320.690661:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768320.690664:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768320/real 1775768320] req@ffff994207889880 x1862022294301568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768320.690673:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768320.690675:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768320.690677:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768320.690679:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768320.690680:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768320.690682:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768320.690685:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207889880 x1862022294301568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768320.690694:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207889880 x1862022294301568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768320.690700:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768320.690701:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768320.690703:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768320.690704:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768320.690707:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207889880 x1862022294301568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768320.690713:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768320.690716:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207889880 x1862022294301568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768345 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768320.690723:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768320.690725:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768320.690726:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768320.690727:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768320.690728:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768320.690731:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768320.690732:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768320.690735:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768320.690736:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768320.690739:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207889880 x1862022294301568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768345 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768320.690744:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207889880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294301568:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768320.690746:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768320.690748:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff99420788b100 x1862022294301696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768320.690751:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768320.690752:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768320.690754:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768320.690755:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768320.690756:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768320.690758:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff99420788b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294301696:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768320.690760:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768320.690761:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768320.690762:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768320.690763:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768320.690765:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1d000. 02000000:00000001:2.0:1775768320.690766:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768320.690768:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768320.690770:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768320.690772:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768320.690774:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294301696, portal 10 00000100:00000001:2.0:1775768320.690775:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768320.690776:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606130944 : -112966103420672 : ffff99420788b100) 00000100:00000040:2.0:1775768320.690780:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff99420788b100 x1862022294301696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768345 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768320.690784:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768320.690785:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768320.690787:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768320.690788:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022294301696, offset 0 00000400:00000200:2.0:1775768320.690806:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768320.690810:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768320.690813:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768320.690816:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768320.690818:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768320.690820:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768320.690823:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768320.690826:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225530:225530:256:4294967295] 192.168.202.156@tcp LPNI seq info [225526:225526:8:4294967295] 00000400:00000200:2.0:1775768320.690829:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768320.690835:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768320.690839:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768320.690843:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768320.690847:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768320.690849:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775768320.690853:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768320.690856:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000100:2.0:1775768320.690860:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768320.690862:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768320.690865:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768320.690868:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768320.690870:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768320.690872:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768320.690874:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768320.690877:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff99420788b100 x1862022294301696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768345 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768320.690884:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768320.690886:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff99420788b100 x1862022294301696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1775768345 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768320.690892:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768320.690893:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768320.690896:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768320.690899:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: using a timeout of 9456 usecs (10 jiffies) 00000400:00000200:2.0:1775768320.690902:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768320.690904:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300245515 00000100:00000001:2.0:1775768320.690907:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300246845 00000100:00000001:2.0:1775768320.690909:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300245515 00000100:00000001:2.0:1775768320.690911:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300246845 00000100:00000001:2.0:1775768320.690913:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768320.690915:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768320.690917:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768320.690921:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768320/real 1775768320] req@ffff99420788b100 x1862022294301696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768320.690929:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768320.690930:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768320.690932:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768320.690934:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768320.690936:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768320.690938:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768320.690940:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff99420788b100 x1862022294301696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768320.690945:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff99420788b100 x1862022294301696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768320.690949:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768320.690950:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768320.690951:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768320.690952:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768320.690953:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff99420788b100 x1862022294301696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768320.690957:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768320.690959:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff99420788b100 x1862022294301696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768345 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768320.690963:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768320.690965:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768320.690966:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768320.690967:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768320.690968:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768320.690970:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768320.690971:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768320.690973:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768320.690974:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768320.690976:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff99420788b100 x1862022294301696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768345 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768320.690981:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff99420788b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294301696:192.168.202.156@tcp:38:kworker.0 00000100:00000001:2.0:1775768320.690984:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768320.690985:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207889f80 x1862022294301824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768320.690989:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768320.690989:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768320.690991:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768320.690992:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768320.690993:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768320.690994:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207889f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294301824:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768320.690996:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768320.690997:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768320.690998:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768320.690999:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768320.691001:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f000. 02000000:00000001:2.0:1775768320.691003:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768320.691005:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768320.691007:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768320.691009:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768320.691011:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294301824, portal 4 00000100:00000001:2.0:1775768320.691013:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768320.691013:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606126464 : -112966103425152 : ffff994207889f80) 00000100:00000040:2.0:1775768320.691017:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207889f80 x1862022294301824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768345 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768320.691021:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768320.691022:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768320.691023:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768320.691025:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294301824, offset 0 00000400:00000200:2.0:1775768320.691027:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768320.691029:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768320.691031:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768320.691033:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768320.691034:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768320.691036:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768320.691037:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768320.691039:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225531:225531:256:4294967295] 192.168.202.156@tcp LPNI seq info [225527:225527:8:4294967295] 00000400:00000200:2.0:1775768320.691043:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768320.691048:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768320.691052:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768320.691055:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768320.691059:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768320.691061:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775768320.691065:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768320.691067:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000100:2.0:1775768320.691070:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768320.691072:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768320.691075:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768320.691078:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768320.691081:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768320.691082:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768320.691085:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768320.691088:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207889f80 x1862022294301824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768345 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768320.691101:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768320.691104:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207889f80 x1862022294301824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768345 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768320.691109:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768320.691110:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768320.691112:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768320.691115:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: using a timeout of 32200 usecs (33 jiffies) 00000400:00000200:2.0:1775768320.691118:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768320.691120:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300245538 00000100:00000001:2.0:1775768320.691123:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300246845 00000100:00000001:2.0:1775768320.691125:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300245538 00000100:00000001:2.0:1775768320.691126:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300246845 00000100:00000001:2.0:1775768320.691129:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768320.691131:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768320.691132:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768320.691136:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768320/real 1775768320] req@ffff994207889f80 x1862022294301824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768320.691143:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768320.691144:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768320.691146:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768320.691148:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768320.691150:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768320.691152:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768320.691175:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207889f80 x1862022294301824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768320.691181:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207889f80 x1862022294301824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768320.691185:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768320.691187:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768320.691188:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768320.691189:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768320.691191:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207889f80 x1862022294301824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768320.691195:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768320.691197:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207889f80 x1862022294301824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768345 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768320.691201:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768320.691203:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768320.691203:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768320.691204:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768320.691205:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768320.691208:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768320.691209:0:2399:0:(import.c:65:import_set_state_nolock()) ffff99421033e800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768320.691211:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768320.691212:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768320.691214:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207889f80 x1862022294301824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768345 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768320.691218:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207889f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294301824:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768320.691220:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768320.691222:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768320.691223:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207889880 x1862022294301568/t0(0) o38->lustre-MDT0000-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768345 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768320.691227:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768320.691228:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768320.691230:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1ec00. 02000000:00000001:2.0:1775768320.691232:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768320.691233:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768320.691234:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033b800 refcount=23 obd=lustre-MDT0000-mdc-ffff9942077a7000 00000020:00000001:2.0:1775768320.691235:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768320.691237:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543e800. 02000000:00000001:2.0:1775768320.691239:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768320.691240:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768320.691241:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207889880. 00000100:00000001:2.0:1775768320.691242:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768320.691243:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768320.691245:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768320.691246:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff99420788b100 x1862022294301696/t0(0) o38->lustre-MDT0001-mdc-ffff9942077a7000@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768345 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768320.691250:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768320.691251:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768320.691251:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1d000. 02000000:00000001:2.0:1775768320.691254:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768320.691254:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768320.691255:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e000 refcount=66 obd=lustre-MDT0001-mdc-ffff9942077a7000 00000020:00000001:2.0:1775768320.691256:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768320.691256:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543f000. 02000000:00000001:2.0:1775768320.691257:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768320.691258:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768320.691259:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff99420788b100. 00000100:00000001:2.0:1775768320.691260:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768320.691261:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768320.691261:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768320.691263:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207889f80 x1862022294301824/t0(0) o8->lustre-OST0000-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768345 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768320.691266:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768320.691266:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768320.691267:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f000. 02000000:00000001:2.0:1775768320.691268:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768320.691269:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768320.691269:0:2399:0:(genops.c:1242:class_import_put()) import ffff99421033e800 refcount=16 obd=lustre-OST0000-osc-ffff9942077a7000 00000020:00000001:2.0:1775768320.691270:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768320.691271:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543f800. 02000000:00000001:2.0:1775768320.691272:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768320.691273:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768320.691274:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207889f80. 00000100:00000001:2.0:1775768320.691275:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768320.691276:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768320.691277:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768320.691280:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768320.691281:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768320.691282:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768320.691283:0:2399:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775768320.691284:0:2399:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775768320.691286:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff994207888e00 x1862022294301952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768320.691291:0:2399:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775768320.691292:0:2399:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768320.691293:0:2399:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775768320.691294:0:2399:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775768320.691295:0:2399:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775768320.691297:0:2399:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff994207888e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294301952:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768320.691298:0:2399:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775768320.691299:0:2399:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775768320.691300:0:2399:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775768320.691301:0:2399:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775768320.691303:0:2399:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff994202d1f000. 02000000:00000001:2.0:1775768320.691304:0:2399:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775768320.691306:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000010:2.0:1775768320.691308:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff994207b39ec8. 00000400:00000010:2.0:1775768320.691310:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f2a8. 00000100:00000200:2.0:1775768320.691312:0:2399:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022294301952, portal 4 00000100:00000001:2.0:1775768320.691313:0:2399:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775768320.691314:0:2399:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446631107606121984 : -112966103429632 : ffff994207888e00) 00000100:00000040:2.0:1775768320.691318:0:2399:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff994207888e00 x1862022294301952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768345 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768320.691322:0:2399:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775768320.691323:0:2399:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:2.0:1775768320.691325:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff994202d8f000. 00000100:00000200:2.0:1775768320.691326:0:2399:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862022294301952, offset 0 00000400:00000200:2.0:1775768320.691329:0:2399:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:2.0:1775768320.691331:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768320.691333:0:2399:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:2.0:1775768320.691335:0:2399:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775768320.691337:0:2399:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:2.0:1775768320.691339:0:2399:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775768320.691340:0:2399:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:2.0:1775768320.691342:0:2399:0:(lib-move.c:1911:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [225532:225532:256:4294967295] 192.168.202.156@tcp LPNI seq info [225528:225528:8:4294967295] 00000400:00000200:2.0:1775768320.691344:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000400:00000200:2.0:1775768320.691348:0:2399:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:2.0:1775768320.691351:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768320.691353:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff994208bd3100. 00000800:00000200:2.0:1775768320.691355:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.0:1775768320.691357:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775768320.691359:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff99421195fc00] -> 12345-192.168.202.156@tcp (2) 00000800:00000200:2.1:1775768320.691361:0:2399:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.156 (cnted 0, interval 60, 42 secs later) 00000800:00000100:2.0:1775768320.691363:0:2399:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.156@tcp 00000800:00000010:2.0:1775768320.691365:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff994208bd3100. 00000400:00000200:2.0:1775768320.691366:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775768320.691368:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775768320.691370:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f000 00000400:00000010:2.0:1775768320.691371:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f000. 00000100:00000001:2.0:1775768320.691373:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775768320.691374:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff994207888e00 x1862022294301952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768345 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768320.691379:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768320.691381:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff994207888e00 x1862022294301952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1775768345 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768320.691384:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768320.691385:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775768320.691387:0:2399:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775768320.691389:0:2399:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: using a timeout of 35968 usecs (36 jiffies) 00000400:00000200:2.0:1775768320.691391:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.202.156@tcp to cpt 0 00000100:00000001:2.0:1775768320.691392:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 0 is 4300245542 00000100:00000001:2.0:1775768320.691394:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 0, keep current deadline 4300246845 00000100:00000001:2.0:1775768320.691395:0:2399:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS new deadline estimation for cpu 1 is 4300245542 00000100:00000001:2.0:1775768320.691396:0:2399:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9942077a7000: PM QoS request active for cpu 1, keep current deadline 4300246845 00000100:00000001:2.0:1775768320.691397:0:2399:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775768320.691398:0:2399:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768320.691399:0:2399:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775768320.691401:0:2399:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768320/real 1775768320] req@ffff994207888e00 x1862022294301952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775768320.691405:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775768320.691406:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775768320.691407:0:2399:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff994207b39ec8. 00000400:00000200:2.0:1775768320.691408:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff994202d8f2a8 00000400:00000010:2.0:1775768320.691409:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff994202d8f2a8. 00000100:00000001:2.0:1775768320.691410:0:2399:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775768320.691412:0:2399:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff994207888e00 x1862022294301952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775768320.691416:0:2399:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff994207888e00 x1862022294301952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768320.691420:0:2399:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775768320.691421:0:2399:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768320.691422:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768320.691423:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775768320.691424:0:2399:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff994207888e00 x1862022294301952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768320.691428:0:2399:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775768320.691429:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff994207888e00 x1862022294301952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768345 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768320.691434:0:2399:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775768320.691435:0:2399:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768320.691436:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775768320.691438:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768320.691439:0:2399:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775768320.691442:0:2399:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775768320.691444:0:2399:0:(import.c:65:import_set_state_nolock()) ffff994206081800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768320.691447:0:2399:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:2.0:1775768320.691449:0:2399:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775768320.691453:0:2399:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff994207888e00 x1862022294301952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768345 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768320.691460:0:2399:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff994207888e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c72e6f0-c073-4915-a1e7-5dde21a8c485:2399:1862022294301952:192.168.202.156@tcp:8:kworker.0 00000100:00000001:2.0:1775768320.691463:0:2399:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768320.691465:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775768320.691468:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff994207888e00 x1862022294301952/t0(0) o8->lustre-OST0001-osc-ffff9942077a7000@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768345 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775768320.691473:0:2399:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775768320.691475:0:2399:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775768320.691476:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff994202d1f000. 02000000:00000001:2.0:1775768320.691479:0:2399:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775768320.691480:0:2399:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775768320.691481:0:2399:0:(genops.c:1242:class_import_put()) import ffff994206081800 refcount=17 obd=lustre-OST0001-osc-ffff9942077a7000 00000020:00000001:2.0:1775768320.691483:0:2399:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775768320.691485:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff99420543fc00. 02000000:00000001:2.0:1775768320.691487:0:2399:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775768320.691488:0:2399:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775768320.691489:0:2399:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff994207888e00. 00000100:00000001:2.0:1775768320.691492:0:2399:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775768320.691493:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768320.691495:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775768320.691498:0:2399:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775768320.691499:0:2399:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775768320.691500:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775768320.691501:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768320.753299:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768320.753304:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768320.753308:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768320.753309:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768320.753312:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768320.753313:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768320.753315:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768320.753316:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775768320.753318:0:15036:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775768320.753319:0:15036:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775768320.753342:0:14635:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775768320.753343:0:14635:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775768320.753345:0:14635:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775768320.817250:0:14452:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775768320.817258:0:14452:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775768320.817261:0:14452:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775768320.817267:0:14452:0:(client.c:2691:ptlrpc_set_wait()) set ffff994207ef4900 going to sleep for 0 seconds 00000100:00000001:1.0:1775768320.817272:0:14452:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775768320.817277:0:14452:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768320.817280:0:14452:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775768320.817283:0:14452:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775768320.817286:0:14452:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) Debug log: 139598 lines, 139598 kept, 0 dropped, 0 bad.